Merge pull request #96 from kalessin/cookiesmultijar

allow to work with multiple cookie jars on the same spider

scrapy/contrib/downloadermiddleware/cookies.py

@@ -1,8 +1,6 @@
 import os
 from collections import defaultdict
-from scrapy.xlib.pydispatch import dispatcher
 
-from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
@@ -18,13 +16,13 @@ class CookiesMiddleware(object):
         if not settings.getbool('COOKIES_ENABLED'):
             raise NotConfigured
         self.jars = defaultdict(CookieJar)
-        dispatcher.connect(self.spider_closed, signals.spider_closed)
 
     def process_request(self, request, spider):
         if 'dont_merge_cookies' in request.meta:
             return
 
-        jar = self.jars[spider]
+        cookiejarkey = request.meta.get("cookiejar")
+        jar = self.jars[cookiejarkey]
         cookies = self._get_request_cookies(jar, request)
         for cookie in cookies:
             jar.set_cookie_if_ok(cookie, request)
@@ -39,15 +37,13 @@ class CookiesMiddleware(object):
             return response
 
         # extract cookies from Set-Cookie and drop invalid/expired cookies
-        jar = self.jars[spider]
+        cookiejarkey = request.meta.get("cookiejar")
+        jar = self.jars[cookiejarkey]
         jar.extract_cookies(response, request)
         self._debug_set_cookie(response, spider)
 
         return response
 
-    def spider_closed(self, spider):
-        self.jars.pop(spider, None)
-
     def _debug_cookie(self, request, spider):
         if self.debug:
             cl = request.headers.getlist('Cookie')

scrapy/tests/test_downloadermiddleware_cookies.py

@@ -14,7 +14,6 @@ class CookiesMiddlewareTest(TestCase):
         self.mw = CookiesMiddleware()
 
     def tearDown(self):
-        self.mw.spider_closed(self.spider)
         del self.mw
 
     def test_basic(self):
@@ -65,6 +64,28 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req2, self.spider) is None
         self.assertEquals(req2.headers.get('Cookie'), "C1=value1; galleta=salada")
 
+    def test_cookiejar_key(self):
+        req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEquals(req.headers.get('Cookie'), 'galleta=salada')
+
+        headers = {'Set-Cookie': 'C1=value1; path=/'}
+        res = Response('http://scrapytest.org/', headers=headers, request=req)
+        assert self.mw.process_response(req, res, self.spider) is res
+
+        req2 = Request('http://scrapytest.org/', meta=res.meta)
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertEquals(req2.headers.get('Cookie'), 'C1=value1; galleta=salada')
 
 
+        req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertEquals(req3.headers.get('Cookie'), 'galleta=dulce')
 
+        headers = {'Set-Cookie': 'C2=value2; path=/'}
+        res2 = Response('http://scrapytest.org/', headers=headers, request=req3)
+        assert self.mw.process_response(req3, res2, self.spider) is res2
+
+        req4 = Request('http://scrapytest.org/', meta=res2.meta)
+        assert self.mw.process_request(req4, self.spider) is None
+        self.assertEquals(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce')