From 3de3c2a331c5fcd221d0e317f0bb74c654e6d5fa Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 21 Apr 2009 17:13:12 +0000 Subject: [PATCH] cookies: fix merging user cookies --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401080 --- .../downloadermiddleware/cookies.py | 19 ++++++++++++++++--- .../test_downloadermiddleware_cookies.py | 4 ++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py b/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py index f4ae7447c..21e3ece04 100644 --- a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py +++ b/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/cookies.py @@ -1,12 +1,15 @@ +import operator +from itertools import groupby +from collections import defaultdict from pydispatch import dispatcher -from collections import defaultdict - from scrapy.core import signals +from scrapy.http import Response from scrapy.utils.cookies import CookieJar from scrapy.core.exceptions import HttpException from scrapy import log + class CookiesMiddleware(object): """This middleware enables working with sites that need cookies""" @@ -19,7 +22,8 @@ class CookiesMiddleware(object): return jar = self.jars[spider.domain_name] - for name, cookie in request.cookies.items(): + cookies = _get_cookies(jar, request) + for cookie in cookies: jar.set_cookie_if_ok(cookie, request) # set Cookie header @@ -43,3 +47,12 @@ class CookiesMiddleware(object): def domain_closed(self, domain): self.jars.pop(domain, None) + + +def _get_cookies(jar, request): + headers = {'Set-Cookie': ['%s=%s;' % (k, v) for k, v in request.cookies.iteritems()]} + response = Response(request.url, headers=headers) + cookies = jar.make_cookies(response, request) + return cookies + + diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py index 333c8883c..e56fe8240 100644 --- a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py @@ -73,16 +73,16 @@ class CookiesMiddlewareTest(TestCase): self.assertEquals(req.headers.get('Cookie'), 'C1=value1') def test_merge_request_cookies(self): - headers = {'Set-Cookie': 'C1=value1; path=/'} req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}) self.mw.process_request(req, self.spider) self.assertEquals(req.headers.get('Cookie'), 'galleta=salada') + headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers) self.mw.process_response(req, res, self.spider) req2 = Request('http://scrapytest.org/sub1/') self.mw.process_request(req2, self.spider) - self.assertEquals(req2.headers.get('Cookie'), "C1=value1") + self.assertEquals(req2.headers.get('Cookie'), "C1=value1; galleta=salada")