From 6ba6238c83a8c1abd4b7ded833d3eafc7e3265a2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 15 Jan 2009 03:24:48 +0000 Subject: [PATCH] Response class: * added meta and cache attributes to Response class * added tests for Response copy Request class: * added meta attribute and renamed old _cache attribute to cache * moved depth and link_text to Request.meta * added tests for Request copy * ResponseLibxml2 and ResponseSoup extensions now use Response.cache Updated doc with changes --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40734 --- scrapy/trunk/docs/ref/request-response.rst | 49 +++++++++++++++---- scrapy/trunk/scrapy/contrib/response/soup.py | 6 +-- .../scrapy/contrib/spidermiddleware/depth.py | 14 +++--- scrapy/trunk/scrapy/contrib/spiders/crawl.py | 3 +- scrapy/trunk/scrapy/http/request.py | 25 ++++++---- scrapy/trunk/scrapy/http/response.py | 10 ++++ .../trunk/scrapy/tests/test_http_request.py | 15 ++++++ .../trunk/scrapy/tests/test_http_response.py | 23 +++++++-- .../trunk/scrapy/tests/test_utils_request.py | 2 +- scrapy/trunk/scrapy/utils/request.py | 4 +- scrapy/trunk/scrapy/xpath/extension.py | 8 +-- 11 files changed, 119 insertions(+), 40 deletions(-) diff --git a/scrapy/trunk/docs/ref/request-response.rst b/scrapy/trunk/docs/ref/request-response.rst index 8c6a5b955..1b050f277 100644 --- a/scrapy/trunk/docs/ref/request-response.rst +++ b/scrapy/trunk/docs/ref/request-response.rst @@ -32,16 +32,39 @@ Attributes .. attribute:: Request.headers - A dict of the request headers. + A dictionary-like object which contains the request headers. .. attribute:: Request.body - The body of the request as a string + A string that contains the request body + +.. attribute:: Request.meta + + A dict that contains arbitrary metadata for this request. This dict is + empty for new Requests, and is usually populated by different Scrapy + components (extensions, middlewares, etc). So the data contained in this + dict depends on the extensions you have enabled. + + This dict is `shallow copied`_ when the request is cloned using the + ``copy()`` or ``replace()`` methods. + +.. _shallow copied: http://docs.python.org/library/copy.html + +.. attribute:: Request.cache + + A dict that contains arbitrary cached data for this request. This dict is + empty for new Requests, and is usually populated by different Scrapy + components (extensions, middlewares, etc) to avoid duplicate processing. So + the data contained in this dict depends on the extensions you have enabled. + + Unlike the ``meta`` attribute, this dict is not copied at all when the + request is cloned using the ``copy()`` or ``replace()`` methods. + Methods ------- -.. method:: Request.__init__(url, callback=None, context=None, method='GET', body=None, headers=None, cookies=None, url_encoding='utf-8', link_text='', dont_filter=None) +.. method:: Request.__init__(url, callback=None, context=None, method='GET', body=None, headers=None, cookies=None, url_encoding='utf-8', dont_filter=None) Instantiates a ``Request`` object with the given arguments: @@ -50,7 +73,7 @@ Methods ``callback`` is a function that will be called with the response of this request (once its downloaded) as its first parameter - ``context`` can be a dict which will be accesible in the callback function + ``context`` can be a dict which will be accessible in the callback function in ``response.request.context`` in the callback function ``method`` is a string with the HTTP method of this request @@ -66,10 +89,6 @@ Methods The request URL will be percent encoded using this encoding before downloading - ``link_text`` is a string describing the URL of this requests. For example, - in ``Example site`` the ``link_text`` - would be ``"Example site"`` - ``dont_filter`` is a boolean which indicates that this request should not be filtered by the scheduler. This is used when you want to perform an identical request multiple times, for whatever reason @@ -89,7 +108,19 @@ Attributes .. attribute:: Response.headers - A dict of the response headers. + A dictionary-like object which contains the response headers. + +.. attribute:: Response.meta + + A dict that contains arbitrary metadata fro this response. It works like + :attr:`Request.meta` for Request objects. See that attribute help for more + info. + +.. attribute:: Response.cache + + A dict that contains arbitrary cached data for this response. It works like + :attr:`Request.cache` for Request objects. See that attribute help for more + info. Methods ------- diff --git a/scrapy/trunk/scrapy/contrib/response/soup.py b/scrapy/trunk/scrapy/contrib/response/soup.py index 19eea6833..a090cc098 100644 --- a/scrapy/trunk/scrapy/contrib/response/soup.py +++ b/scrapy/trunk/scrapy/contrib/response/soup.py @@ -16,7 +16,7 @@ class ResponseSoup(object): def getsoup(response, **kwargs): # TODO: use different cache buckets depending on constructor parameters - if not hasattr(response, '_soup'): + if 'soup' not in response.cache: body = response.body.to_string() if response.body is not None else "" - setattr(response, '_soup', BeautifulSoup(body, **kwargs)) - return response._soup + response.cache['soup'] = BeautifulSoup(body, **kwargs) + return response.cache['soup'] diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py index 81e5e4cd4..fd0f4190e 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py @@ -20,17 +20,19 @@ class DepthMiddleware(object): def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request): - request.depth = response.request.depth + 1 - if self.maxdepth and request.depth > self.maxdepth: + depth = response.request.meta['depth'] + 1 + request.meta['depth'] = depth + if self.maxdepth and depth > self.maxdepth: log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name) return False elif self.stats: - stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, request.depth)) - if request.depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0): - stats.setpath('%s/request_depth_max' % spider.domain_name, request.depth) + stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, depth)) + if depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0): + stats.setpath('%s/request_depth_max' % spider.domain_name, depth) return True - if self.stats and response.request.depth == 0: # otherwise we loose stats for depth=0 + if self.stats and 'depth' not in response.request.meta: # otherwise we loose stats for depth=0 + response.request.meta['depth'] = 0 stats.incpath('%s/request_depth_count/0' % spider.domain_name) return (r for r in result or () if _filter(r)) diff --git a/scrapy/trunk/scrapy/contrib/spiders/crawl.py b/scrapy/trunk/scrapy/contrib/spiders/crawl.py index ff28a4dcf..25cf3d5fc 100644 --- a/scrapy/trunk/scrapy/contrib/spiders/crawl.py +++ b/scrapy/trunk/scrapy/contrib/spiders/crawl.py @@ -90,7 +90,8 @@ class CrawlSpider(BaseSpider): links = rule.process_links(links) seen = seen.union(links) for link in links: - r = Request(url=link.url, link_text=link.text) + r = Request(url=link.url) + r.meta['link_text'] = link.text r.append_callback(self._response_downloaded, rule.callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow) requests.append(r) return requests diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py index eb9420fe5..a3c777ec2 100644 --- a/scrapy/trunk/scrapy/http/request.py +++ b/scrapy/trunk/scrapy/http/request.py @@ -1,5 +1,12 @@ +""" +This module implements the Request class which is used to represent HTTP +requests in Scrapy. + +See documentation in docs/ref/request-response.rst +""" + import urllib -from copy import copy +import copy from twisted.internet import defer @@ -12,7 +19,7 @@ class Request(object): def __init__(self, url, callback=None, context=None, method='GET', body=None, headers=None, cookies=None, - url_encoding='utf-8', link_text='', dont_filter=None, domain=None): + url_encoding='utf-8', dont_filter=None, domain=None): self.encoding = url_encoding # this one has to be set first self.set_url(url) @@ -37,13 +44,11 @@ class Request(object): self.context = context or {} # dont_filter be filtered by scheduler self.dont_filter = dont_filter - self.depth = 0 - self.link_text = link_text #allows to directly specify the spider for the request self.domain = domain - # bucket to store cached data such as fingerprint and others - self._cache = {} + self.meta = {} + self.cache = {} def append_callback(self, callback, *args, **kwargs): if isinstance(callback, defer.Deferred): @@ -79,12 +84,12 @@ class Request(object): def copy(self): """Clone request except `context` attribute""" - new = copy(self) - new._cache = {} + new = copy.copy(self) + new.cache = {} for att in self.__dict__: - if att not in ['_cache', 'context', 'url', 'deferred']: + if att not in ['cache', 'context', 'url', 'deferred']: value = getattr(self, att) - setattr(new, att, copy(value)) + setattr(new, att, copy.copy(value)) new.deferred = defer.Deferred() new.context = self.context # requests shares same context dictionary return new diff --git a/scrapy/trunk/scrapy/http/response.py b/scrapy/trunk/scrapy/http/response.py index aab874669..9566d88b5 100644 --- a/scrapy/trunk/scrapy/http/response.py +++ b/scrapy/trunk/scrapy/http/response.py @@ -1,3 +1,10 @@ +""" +This module implements the Response class which is used to represent HTTP +responses in Scrapy. + +See documentation in docs/ref/request-response.rst +""" + import re import hashlib import copy @@ -34,6 +41,8 @@ class Response(object) : self.body = ResponseBody(body, self.headers_encoding()) self.cached = False self.request = None # request which originated this response + self.meta = {} + self.cache = {} def version(self): """A hash of the contents of this response""" @@ -85,6 +94,7 @@ class Response(object) : # Response.__init__ forbids the use of ResponseBody instances if 'body' not in kw: newresp.body = samebody() + newresp.meta = self.meta.copy() return newresp def to_string(self): diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py index 07782d0fd..b9b6d793c 100644 --- a/scrapy/trunk/scrapy/tests/test_http_request.py +++ b/scrapy/trunk/scrapy/tests/test_http_request.py @@ -79,5 +79,20 @@ class RequestTest(unittest.TestCase): self.assert_(isinstance(r.url, str)) self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + def test_copy(self): + """Test Request copy""" + + r1 = Request("http://www.example.com") + r1.meta['foo'] = 'bar' + r1.cache['lala'] = 'lolo' + r2 = r1.copy() + + assert r1.cache + assert not r2.cache + + # make sure meta dict is shallow copied + assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" + self.assertEqual(r1.meta, r2.meta) + if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_http_response.py b/scrapy/trunk/scrapy/tests/test_http_response.py index 2d7d0641f..9bf41ce78 100644 --- a/scrapy/trunk/scrapy/tests/test_http_response.py +++ b/scrapy/trunk/scrapy/tests/test_http_response.py @@ -1,7 +1,7 @@ -from unittest import TestCase, main +import unittest from scrapy.http import Response, ResponseBody -class ResponseTest(TestCase): +class ResponseTest(unittest.TestCase): def test_init(self): # Response requires domain and url self.assertRaises(Exception, Response) @@ -14,7 +14,22 @@ class ResponseTest(TestCase): # test presence of all optional parameters self.assertTrue(isinstance(Response('example.com', 'http://example.com/', original_url='http://example.com/None', headers={}, status=200, body=None), Response)) -class ResponseBodyTest(TestCase): + def test_copy(self): + """Test Response copy""" + + r1 = Response('example.com', "http://www.example.com") + r1.meta['foo'] = 'bar' + r1.cache['lala'] = 'lolo' + r2 = r1.copy() + + assert r1.cache + assert not r2.cache + + # make sure meta dict is shallow copied + assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" + self.assertEqual(r1.meta, r2.meta) + +class ResponseBodyTest(unittest.TestCase): unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442' def test_encoding(self): @@ -34,4 +49,4 @@ class ResponseBodyTest(TestCase): self.assertEqual(cp1251_body.to_string('utf-8'), self.unicode_string.encode('utf-8')) if __name__ == "__main__": - main() + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_utils_request.py b/scrapy/trunk/scrapy/tests/test_utils_request.py index 6ecc07d51..57786b1b5 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_request.py +++ b/scrapy/trunk/scrapy/tests/test_utils_request.py @@ -15,7 +15,7 @@ class UtilsRequestTest(unittest.TestCase): self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) # make sure caching is working - self.assertEqual(request_fingerprint(r1), r1._cache['fingerprint']) + self.assertEqual(request_fingerprint(r1), r1.cache['fingerprint']) r1 = Request("http://www.example.com/members/offers.html") r2 = Request("http://www.example.com/members/offers.html") diff --git a/scrapy/trunk/scrapy/utils/request.py b/scrapy/trunk/scrapy/utils/request.py index e9da7c9cf..b219808d3 100644 --- a/scrapy/trunk/scrapy/utils/request.py +++ b/scrapy/trunk/scrapy/utils/request.py @@ -43,7 +43,7 @@ def request_fingerprint(request, include_headers=()): cachekey = 'fingerprint' try: - return request._cache[cachekey] + return request.cache[cachekey] except KeyError: fp = hashlib.sha1() fp.update(request.method) @@ -54,7 +54,7 @@ def request_fingerprint(request, include_headers=()): fp.update(hdr) fp.update(request.headers.get(hdr, '')) fphash = fp.hexdigest() - request._cache[cachekey] = fphash + request.cache[cachekey] = fphash return fphash def request_authenticate(request, username, password): diff --git a/scrapy/trunk/scrapy/xpath/extension.py b/scrapy/trunk/scrapy/xpath/extension.py index 0642e0cd6..fc07689c2 100644 --- a/scrapy/trunk/scrapy/xpath/extension.py +++ b/scrapy/trunk/scrapy/xpath/extension.py @@ -12,9 +12,9 @@ class ResponseLibxml2(object): setattr(Response, 'getlibxml2doc', getlibxml2doc) def getlibxml2doc(response, constructor=xmlDoc_from_html): - attr = '_lx2doc_%s' % constructor.__name__ - if not hasattr(response, attr): + cachekey = 'lx2doc_%s' % constructor.__name__ + if cachekey not in response.cache: lx2doc = Libxml2Document(response, constructor=constructor) - setattr(response, attr, lx2doc) - return getattr(response, attr) + response.cache[cachekey] = lx2doc + return response.cache[cachekey]