cookies: add new cookie middleware to experimental

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401065

scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py

@@ -0,0 +1,97 @@
+from pydispatch import dispatcher
+
+from collections import defaultdict
+from cookielib import CookieJar
+
+from scrapy.core import signals
+from scrapy.utils.misc import dict_updatedefault
+from scrapy import log
+
+class CookiesMiddleware(object):
+    """This middleware enables working with sites that need cookies"""
+
+    def __init__(self):
+        self.cookies = defaultdict(self.defaultdict_factory)
+        dispatcher.connect(self.domain_closed, signals.domain_closed)
+
+    def process_request(self, request, spider):
+        if request.meta.get('dont_merge_cookies', False):
+            return
+        jar = self.cookies[spider.domain_name]
+        wreq = _WrappedRequest(request)
+
+        for key, value in request.cookies.items():
+            jar.set_cookie
+
+
+        jar.add_cookie_header(wreq)
+        print request.url.netloc, '    Cookie: ', request.headers.get('Cookie'), request.cookies
+
+    def process_response(self, request, response, spider):
+        print request.url.netloc, 'Set-Cookie:', response.headers.get('Set-Cookie'), request.cookies
+        wreq = _WrappedRequest(request)
+        wrsp = _WrappedResponse(response)
+        jar = self.cookies[spider.domain_name]
+        jar.extract_cookies(wrsp, wreq)
+        return response
+
+    def domain_closed(self, domain):
+        if domain in self.cookies:
+            del self.cookies[domain]
+
+    def defaultdict_factory(self):
+        return CookieJar()
+
+
+class _WrappedRequest(object):
+    """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
+
+    see http://docs.python.org/library/urllib2.html#urllib2.Request
+    """
+
+    def __init__(self, request):
+        self.request = request
+
+    def get_full_url(self):
+        return self.request.url
+
+    def get_host(self):
+        return self.request.url.netloc
+
+    def get_type(self):
+        return self.request.url.scheme
+
+    def is_unverifiable(self):
+        """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965.
+
+        It defaults to False. An unverifiable request is one whose URL the user did not have the
+        option to approve. For example, if the request is for an image in an
+        HTML document, and the user had no option to approve the automatic
+        fetching of the image, this should be true.
+        """
+        return self.request.meta.get('is_unverifiable', False)
+
+    def get_origin_req_host(self):
+        return self.request.hostname
+
+    def has_header(self, name):
+        return name in self.request.headers
+
+    def header_items(self):
+        return self.request.headers.items()
+
+    def add_unredirected_header(self, name, value):
+        # XXX: review please, not sure how to handle this
+        self.request.headers[name] = value
+
+
+class _WrappedResponse(object):
+
+    def __init__(self, response):
+        self.response = response
+
+    def info(self):
+        return self
+
+    def getheaders(self, name):
+        return self.response.headers.getlist(name)