From 512b75942ccf021fcbc1b9150d22f7e89f5742bd Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 16 Apr 2009 19:14:32 +0000 Subject: [PATCH] cookies: add TODO comments to experimental cookie mw --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401066 --- .../downloadermiddleware/cookies.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py b/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py index afacfae94..0cb7b9b3e 100644 --- a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py +++ b/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py @@ -11,37 +11,42 @@ class CookiesMiddleware(object): """This middleware enables working with sites that need cookies""" def __init__(self): - self.cookies = defaultdict(self.defaultdict_factory) + self.cookies = defaultdict(CookieJar) dispatcher.connect(self.domain_closed, signals.domain_closed) def process_request(self, request, spider): if request.meta.get('dont_merge_cookies', False): return + jar = self.cookies[spider.domain_name] wreq = _WrappedRequest(request) - for key, value in request.cookies.items(): - jar.set_cookie - + # TODO: Merge cookies in request with jar here + # for key, value in request.cookies.items(): + # jar.set_cookie(..) + # set Cookie header with cookies in jar jar.add_cookie_header(wreq) + print request.url.netloc, ' Cookie: ', request.headers.get('Cookie'), request.cookies def process_response(self, request, response, spider): print request.url.netloc, 'Set-Cookie:', response.headers.get('Set-Cookie'), request.cookies + + # extract cookies from Set-Cookie and drop invalid/expired cookies wreq = _WrappedRequest(request) wrsp = _WrappedResponse(response) jar = self.cookies[spider.domain_name] + jar.extract_cookies(wrsp, wreq) + + # TODO: set current cookies in jar to response.cookies? return response def domain_closed(self, domain): if domain in self.cookies: del self.cookies[domain] - def defaultdict_factory(self): - return CookieJar() - class _WrappedRequest(object): """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class