From 5861b345c72c817a53148316fdc1dc351af5d186 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 20 Apr 2009 23:59:24 +0000 Subject: [PATCH] cookies: add cookiejar to wrap cookielib and remove cookielib from mw --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401073 --- .../downloadermiddleware/cookies.py | 78 ++----------- scrapy/trunk/scrapy/utils/cookies.py | 107 ++++++++++++++++++ 2 files changed, 118 insertions(+), 67 deletions(-) create mode 100644 scrapy/trunk/scrapy/utils/cookies.py diff --git a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py b/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py index 0cb7b9b3e..36fecf7b5 100644 --- a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py +++ b/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py @@ -1,10 +1,9 @@ from pydispatch import dispatcher from collections import defaultdict -from cookielib import CookieJar from scrapy.core import signals -from scrapy.utils.misc import dict_updatedefault +from scrapy.utils.cookies import CookieJar from scrapy import log class CookiesMiddleware(object): @@ -19,84 +18,29 @@ class CookiesMiddleware(object): return jar = self.cookies[spider.domain_name] - wreq = _WrappedRequest(request) + for name, cookie in request.cookies.items(): + jar.set_cookie_if_ok(cookie, request) - # TODO: Merge cookies in request with jar here - # for key, value in request.cookies.items(): - # jar.set_cookie(..) - - # set Cookie header with cookies in jar + # set Cookie header jar.add_cookie_header(wreq) - print request.url.netloc, ' Cookie: ', request.headers.get('Cookie'), request.cookies - def process_response(self, request, response, spider): - print request.url.netloc, 'Set-Cookie:', response.headers.get('Set-Cookie'), request.cookies + if request.meta.get('dont_merge_cookies', False): + return # extract cookies from Set-Cookie and drop invalid/expired cookies - wreq = _WrappedRequest(request) - wrsp = _WrappedResponse(response) jar = self.cookies[spider.domain_name] - jar.extract_cookies(wrsp, wreq) # TODO: set current cookies in jar to response.cookies? return response + # cookies should be set on non-200 responses too + def process_exception(self, request, exception, spider): + if isinstance(exception, HttpException): + self.process_response(request, exception.response, spider) + def domain_closed(self, domain): if domain in self.cookies: del self.cookies[domain] - -class _WrappedRequest(object): - """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class - - see http://docs.python.org/library/urllib2.html#urllib2.Request - """ - - def __init__(self, request): - self.request = request - - def get_full_url(self): - return self.request.url - - def get_host(self): - return self.request.url.netloc - - def get_type(self): - return self.request.url.scheme - - def is_unverifiable(self): - """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965. - - It defaults to False. An unverifiable request is one whose URL the user did not have the - option to approve. For example, if the request is for an image in an - HTML document, and the user had no option to approve the automatic - fetching of the image, this should be true. - """ - return self.request.meta.get('is_unverifiable', False) - - def get_origin_req_host(self): - return self.request.hostname - - def has_header(self, name): - return name in self.request.headers - - def header_items(self): - return self.request.headers.items() - - def add_unredirected_header(self, name, value): - # XXX: review please, not sure how to handle this - self.request.headers[name] = value - - -class _WrappedResponse(object): - - def __init__(self, response): - self.response = response - - def info(self): - return self - - def getheaders(self, name): - return self.response.headers.getlist(name) diff --git a/scrapy/trunk/scrapy/utils/cookies.py b/scrapy/trunk/scrapy/utils/cookies.py new file mode 100644 index 000000000..e5c5dde97 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/cookies.py @@ -0,0 +1,107 @@ +from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy + + +class CookieJar(object): + def __init__(self, policy=None): + self.jar = _CookieJar(policy or DefaultCookiePolicy()) + + def extract_cookies(self, response, request): + wreq = WrappedRequest(request) + wrsp = WrappedResponse(response) + return self.jar.extract_cookies(wrsp, wreq) + + def add_cookie_header(self, request): + wreq = WrappedRequest(request) + self.jar.add_cookie_header(wreq) + for hdr in ('Cookie', 'Cookie2'): + v = wreq.get_header(hdr) + if hdr: + request.headers[hdr] = v + + + @property + def _cookies(self): + return self.jar._cookies + + def clear_session_cookies(self, *args, **kwargs): + return self.jar.clear_session_cookies(*args, **kwargs) + + def clear(self): + return self.jar.clear() + + def __iter__(self): + return iter(self.jar) + + def __len__(self): + return len(self.jar) + + def set_policy(self, pol): + return self.jar.set_policy(pol) + + def make_cookies(self, response, request): + wreq = WrappedRequest(request) + wrsp = WrappedResponse(response) + return self.jar.make_cookies(wrsp, wreq) + + def set_cookie(self, cookie): + self.jar.set_cookie(cookie) + + def set_cookie_if_ok(self, cookie, request): + self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) + + +class WrappedRequest(object): + """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class + + see http://docs.python.org/library/urllib2.html#urllib2.Request + """ + + def __init__(self, request): + self.request = request + + def get_full_url(self): + return self.request.url + + def get_host(self): + return self.request.url.netloc + + def get_type(self): + return self.request.url.scheme + + def is_unverifiable(self): + """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965. + + It defaults to False. An unverifiable request is one whose URL the user did not have the + option to approve. For example, if the request is for an image in an + HTML document, and the user had no option to approve the automatic + fetching of the image, this should be true. + """ + return self.request.meta.get('is_unverifiable', False) + + def get_origin_req_host(self): + return self.request.hostname + + def has_header(self, name): + return name in self.request.headers + + def get_header(self, name, default=None): + return self.request.headers.get(name, default) + + def header_items(self): + return self.request.headers.items() + + def add_unredirected_header(self, name, value): + self.request.headers[name] = value + #print 'add_unredirected_header', self.request.headers + + +class WrappedResponse(object): + + def __init__(self, response): + self.response = response + + def info(self): + return self + + def getheaders(self, name): + return self.response.headers.getlist(name)