cookies: add cookiejar to wrap cookielib and remove cookielib from mw

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401073

scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py

@@ -1,10 +1,9 @@
 from pydispatch import dispatcher
 
 from collections import defaultdict
-from cookielib import CookieJar
 
 from scrapy.core import signals
-from scrapy.utils.misc import dict_updatedefault
+from scrapy.utils.cookies import CookieJar
 from scrapy import log
 
 class CookiesMiddleware(object):
@@ -19,84 +18,29 @@ class CookiesMiddleware(object):
             return
 
         jar = self.cookies[spider.domain_name]
-        wreq = _WrappedRequest(request)
+        for name, cookie in request.cookies.items():
+            jar.set_cookie_if_ok(cookie, request)
 
-        # TODO: Merge cookies in request with jar here
-        # for key, value in request.cookies.items():
-        #     jar.set_cookie(..)
-
-        # set Cookie header with cookies in jar
+        # set Cookie header
         jar.add_cookie_header(wreq)
 
-        print request.url.netloc, '    Cookie: ', request.headers.get('Cookie'), request.cookies
-
     def process_response(self, request, response, spider):
-        print request.url.netloc, 'Set-Cookie:', response.headers.get('Set-Cookie'), request.cookies
+        if request.meta.get('dont_merge_cookies', False):
+            return
 
         # extract cookies from Set-Cookie and drop invalid/expired cookies
-        wreq = _WrappedRequest(request)
-        wrsp = _WrappedResponse(response)
         jar = self.cookies[spider.domain_name]
-
         jar.extract_cookies(wrsp, wreq)
 
         # TODO: set current cookies in jar to response.cookies?
         return response
 
+    # cookies should be set on non-200 responses too
+    def process_exception(self, request, exception, spider):
+        if isinstance(exception, HttpException):
+            self.process_response(request, exception.response, spider)
+
     def domain_closed(self, domain):
         if domain in self.cookies:
             del self.cookies[domain]
 
-
-class _WrappedRequest(object):
-    """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
-
-    see http://docs.python.org/library/urllib2.html#urllib2.Request
-    """
-
-    def __init__(self, request):
-        self.request = request
-
-    def get_full_url(self):
-        return self.request.url
-
-    def get_host(self):
-        return self.request.url.netloc
-
-    def get_type(self):
-        return self.request.url.scheme
-
-    def is_unverifiable(self):
-        """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965.
-
-        It defaults to False. An unverifiable request is one whose URL the user did not have the
-        option to approve. For example, if the request is for an image in an
-        HTML document, and the user had no option to approve the automatic
-        fetching of the image, this should be true.
-        """
-        return self.request.meta.get('is_unverifiable', False)
-
-    def get_origin_req_host(self):
-        return self.request.hostname
-
-    def has_header(self, name):
-        return name in self.request.headers
-
-    def header_items(self):
-        return self.request.headers.items()
-
-    def add_unredirected_header(self, name, value):
-        # XXX: review please, not sure how to handle this
-        self.request.headers[name] = value
-
-
-class _WrappedResponse(object):
-
-    def __init__(self, response):
-        self.response = response
-
-    def info(self):
-        return self
-
-    def getheaders(self, name):
-        return self.response.headers.getlist(name)

scrapy/trunk/scrapy/utils/cookies.py

@@ -0,0 +1,107 @@
+from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy
+
+
+class CookieJar(object):
+    def __init__(self, policy=None):
+        self.jar = _CookieJar(policy or DefaultCookiePolicy())
+
+    def extract_cookies(self, response, request):
+        wreq = WrappedRequest(request)
+        wrsp = WrappedResponse(response)
+        return self.jar.extract_cookies(wrsp, wreq)
+
+    def add_cookie_header(self, request):
+        wreq = WrappedRequest(request)
+        self.jar.add_cookie_header(wreq)
+        for hdr in ('Cookie', 'Cookie2'):
+            v = wreq.get_header(hdr)
+            if hdr:
+                request.headers[hdr] = v
+
+
+    @property
+    def _cookies(self):
+        return self.jar._cookies
+
+    def clear_session_cookies(self, *args, **kwargs):
+        return self.jar.clear_session_cookies(*args, **kwargs)
+
+    def clear(self):
+        return self.jar.clear()
+
+    def __iter__(self):
+        return iter(self.jar)
+
+    def __len__(self):
+        return len(self.jar)
+
+    def set_policy(self, pol):
+        return self.jar.set_policy(pol)
+
+    def make_cookies(self, response, request):
+        wreq = WrappedRequest(request)
+        wrsp = WrappedResponse(response)
+        return self.jar.make_cookies(wrsp, wreq)
+
+    def set_cookie(self, cookie):
+        self.jar.set_cookie(cookie)
+
+    def set_cookie_if_ok(self, cookie, request):
+        self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))
+
+
+class WrappedRequest(object):
+    """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
+
+    see http://docs.python.org/library/urllib2.html#urllib2.Request
+    """
+
+    def __init__(self, request):
+        self.request = request
+
+    def get_full_url(self):
+        return self.request.url
+
+    def get_host(self):
+        return self.request.url.netloc
+
+    def get_type(self):
+        return self.request.url.scheme
+
+    def is_unverifiable(self):
+        """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965.
+
+        It defaults to False. An unverifiable request is one whose URL the user did not have the
+        option to approve. For example, if the request is for an image in an
+        HTML document, and the user had no option to approve the automatic
+        fetching of the image, this should be true.
+        """
+        return self.request.meta.get('is_unverifiable', False)
+
+    def get_origin_req_host(self):
+        return self.request.hostname
+
+    def has_header(self, name):
+        return name in self.request.headers
+
+    def get_header(self, name, default=None):
+        return self.request.headers.get(name, default)
+
+    def header_items(self):
+        return self.request.headers.items()
+
+    def add_unredirected_header(self, name, value):
+        self.request.headers[name] = value
+        #print 'add_unredirected_header', self.request.headers
+
+
+class WrappedResponse(object):
+
+    def __init__(self, response):
+        self.response = response
+
+    def info(self):
+        return self
+
+    def getheaders(self, name):
+        return self.response.headers.getlist(name)