From e665e5abb70eef0081bfad48abe7efe3a66344f5 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 14 Jun 2010 22:00:54 -0300 Subject: [PATCH 1/4] bumped version to 0.10-dev --- scrapy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 16692e414..fd1628ef4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 9, 0, 'rc1') -__version__ = "0.9-rc1" +version_info = (0, 10, 0, 'dev') +__version__ = "0.10-dev" import sys, os, warnings From b3a65d3313724ab2dc9f3dca0c0b0a025ed68fe5 Mon Sep 17 00:00:00 2001 From: Ping Yin Date: Fri, 9 Jul 2010 13:14:25 -0300 Subject: [PATCH 2/4] HTTPCACHE: Don't cache response with codes in HTTPCACHE_IGNORE_HTTP_CODES --- docs/topics/downloader-middleware.rst | 11 +++++++++++ scrapy/conf/default_settings.py | 1 + .../contrib/downloadermiddleware/httpcache.py | 8 ++++++-- .../test_downloadermiddleware_httpcache.py | 17 +++++++++++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dfd30d3b2..fe0bfeb8a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -276,6 +276,17 @@ Number of seconds to use for HTTP cache expiration. Requests that were cached before this time will be re-downloaded. If zero, cached requests will always expire. A negative number means requests will never expire. +.. setting:: HTTPCACHE_IGNORE_HTTP_CODES + +HTTPCACHE_IGNORE_HTTP_CODES +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.10 + +Default: ``[]`` + +Don't cache response with these HTTP codes. + .. setting:: HTTPCACHE_IGNORE_MISSING HTTPCACHE_IGNORE_MISSING diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 809a62b98..e85f43d11 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -123,6 +123,7 @@ HTTPCACHE_DIR = '' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 +HTTPCACHE_IGNORE_HTTP_CODES = [] ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 341e9b5a1..d1af33d7a 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -22,6 +22,7 @@ class HttpCacheMiddleware(object): def __init__(self, settings=conf.settings): self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') + self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) @@ -35,17 +36,20 @@ class HttpCacheMiddleware(object): if not self.is_cacheable(request): return response = self.storage.retrieve_response(spider, request) - if response: + if response and self.is_cacheable_response(response): response.flags.append('cached') return response elif self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): - if self.is_cacheable(request): + if self.is_cacheable(request) and self.is_cacheable_response(response): self.storage.store_response(spider, request, response) return response + def is_cacheable_response(self, response): + return response.status not in self.ignore_http_codes + def is_cacheable(self, request): return urlparse_cached(request).scheme in ['http', 'https'] diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index b9331110c..230352804 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -24,6 +24,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): settings = { 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, + 'HTTPCACHE_IGNORE_HTTP_CODES': [], } settings.update(new_settings) return Settings(settings) @@ -76,6 +77,22 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags + def test_middleware_ignore_http_codes(self): + # test response is not cached + mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) + assert mw.process_request(self.request, self.spider) is None + mw.process_response(self.request, self.response, self.spider) + assert mw.storage.retrieve_response(self.spider, self.request) is None + assert mw.process_request(self.request, self.spider) is None + + # test response is cached + mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) + mw.process_response(self.request, self.response, self.spider) + response = mw.process_request(self.request, self.spider) + assert isinstance(response, HtmlResponse) + self.assertEqualResponse(self.response, response) + assert 'cached' in response.flags + def assertEqualResponse(self, response1, response2): self.assertEqual(response1.url, response2.url) self.assertEqual(response1.status, response2.status) From b799e5ee373a26c06abd7ee59271722df746a3e2 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 16 Jul 2010 14:51:14 -0300 Subject: [PATCH 3/4] Support default headers per spider. closes #181 --HG-- extra : rebase_source : 60162dffa4fbab525501e46b479dc272b8998942 --- .../downloadermiddleware/defaultheaders.py | 18 ++++++++++++++--- ...est_downloadermiddleware_defaultheaders.py | 20 ++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py index cebfe247b..b3fd3cb37 100644 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -3,15 +3,27 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ - from scrapy.conf import settings +from scrapy.xlib.pydispatch import dispatcher +from scrapy.core import signals + class DefaultHeadersMiddleware(object): def __init__(self): - self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS') + self.global_default_headers = settings.get('DEFAULT_REQUEST_HEADERS') + self._default_headers = {} + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def process_request(self, request, spider): - for k, v in self.default_headers.iteritems(): + for k, v in self._default_headers[spider].iteritems(): if v: request.headers.setdefault(k, v) + + def spider_opened(self, spider): + self._default_headers[spider] = dict(self.global_default_headers, + **getattr(spider, 'default_headers', {})) + + def spider_closed(self, spider): + self._default_headers.pop(spider) diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index 805289787..129495dc7 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.conf import settings from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware -from scrapy.http import Response, Request +from scrapy.http import Request from scrapy.spider import BaseSpider @@ -16,15 +16,33 @@ class TestDefaultHeadersMiddleware(TestCase): def test_process_request(self): req = Request('http://www.scrapytest.org') + self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) + self.mw.spider_closed(self.spider) self.assertEquals(req.headers, self.default_headers) + def test_spider_default_headers(self): + spider_headers = {'Unexistant-Header': ['value']} + # override one of the global default headers by spider + if self.default_headers: + k = set(self.default_headers).pop() + spider_headers[k] = ['__newvalue__'] + self.spider.default_headers = spider_headers + + req = Request('http://www.scrapytest.org') + self.mw.spider_opened(self.spider) + self.mw.process_request(req, self.spider) + self.mw.spider_closed(self.spider) + self.assertEquals(req.headers, dict(self.default_headers, **spider_headers)) + def test_update_headers(self): headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} req = Request('http://www.scrapytest.org', headers=headers) self.assertEquals(req.headers, headers) + self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) + self.mw.spider_closed(self.spider) self.default_headers.update(headers) self.assertEquals(req.headers, self.default_headers) From 3e013f564b95ea9022ce8cff39c2210bcf76e61e Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 16 Jul 2010 16:17:08 -0300 Subject: [PATCH 4/4] update docs for defaultheaders middleware and change spider attribute to match global setting name --- docs/topics/downloader-middleware.rst | 4 +++- .../downloadermiddleware/defaultheaders.py | 2 +- ...test_downloadermiddleware_defaultheaders.py | 18 +++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fe0bfeb8a..77f327b35 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -177,7 +177,9 @@ DefaultHeadersMiddleware .. class:: DefaultHeadersMiddleware This middleware sets all default requests headers specified in the - :setting:`DEFAULT_REQUEST_HEADERS` setting. + :setting:`DEFAULT_REQUEST_HEADERS` setting plus those found in spider + ``default_request_headers`` attribute. Spider headers has precedence over + global headers. HttpAuthMiddleware ------------------ diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py index b3fd3cb37..1fab70858 100644 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -23,7 +23,7 @@ class DefaultHeadersMiddleware(object): def spider_opened(self, spider): self._default_headers[spider] = dict(self.global_default_headers, - **getattr(spider, 'default_headers', {})) + **getattr(spider, 'default_request_headers', {})) def spider_closed(self, spider): self._default_headers.pop(spider) diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index 129495dc7..d24882d02 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -11,7 +11,7 @@ class TestDefaultHeadersMiddleware(TestCase): def setUp(self): self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() - self.default_headers = dict([(k, [v]) for k, v in \ + self.default_request_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) def test_process_request(self): @@ -19,21 +19,21 @@ class TestDefaultHeadersMiddleware(TestCase): self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) self.mw.spider_closed(self.spider) - self.assertEquals(req.headers, self.default_headers) + self.assertEquals(req.headers, self.default_request_headers) - def test_spider_default_headers(self): + def test_spider_default_request_headers(self): spider_headers = {'Unexistant-Header': ['value']} # override one of the global default headers by spider - if self.default_headers: - k = set(self.default_headers).pop() + if self.default_request_headers: + k = set(self.default_request_headers).pop() spider_headers[k] = ['__newvalue__'] - self.spider.default_headers = spider_headers + self.spider.default_request_headers = spider_headers req = Request('http://www.scrapytest.org') self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) self.mw.spider_closed(self.spider) - self.assertEquals(req.headers, dict(self.default_headers, **spider_headers)) + self.assertEquals(req.headers, dict(self.default_request_headers, **spider_headers)) def test_update_headers(self): headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} @@ -43,6 +43,6 @@ class TestDefaultHeadersMiddleware(TestCase): self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) self.mw.spider_closed(self.spider) - self.default_headers.update(headers) - self.assertEquals(req.headers, self.default_headers) + self.default_request_headers.update(headers) + self.assertEquals(req.headers, self.default_request_headers)