Response class:

* added meta and cache attributes to Response class
 * added tests for Response copy

Request class:
 * added meta attribute and renamed old _cache attribute to cache
 * moved depth and link_text to Request.meta
 * added tests for Request copy

* ResponseLibxml2 and ResponseSoup extensions now use Response.cache

Updated doc with changes

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40734

scrapy/trunk/docs/ref/request-response.rst

@@ -32,16 +32,39 @@ Attributes
 
 .. attribute:: Request.headers
 
-    A dict of the request headers.
+    A dictionary-like object which contains the request headers.
 
 .. attribute:: Request.body
 
-    The body of the request as a string
+    A string that contains the request body
+
+.. attribute:: Request.meta
+
+    A dict that contains arbitrary metadata for this request. This dict is
+    empty for new Requests, and is usually  populated by different Scrapy
+    components (extensions, middlewares, etc). So the data contained in this
+    dict depends on the extensions you have enabled.
+
+    This dict is `shallow copied`_ when the request is cloned using the
+    ``copy()`` or ``replace()`` methods.
+
+.. _shallow copied: http://docs.python.org/library/copy.html
+
+.. attribute:: Request.cache
+
+    A dict that contains arbitrary cached data for this request. This dict is
+    empty for new Requests, and is usually populated by different Scrapy
+    components (extensions, middlewares, etc) to avoid duplicate processing. So
+    the data contained in this dict depends on the extensions you have enabled.
+
+    Unlike the ``meta`` attribute, this dict is not copied at all when the
+    request is cloned using the ``copy()`` or ``replace()`` methods.
+
 
 Methods
 -------
 
-.. method:: Request.__init__(url, callback=None, context=None, method='GET', body=None, headers=None, cookies=None, url_encoding='utf-8', link_text='', dont_filter=None)
+.. method:: Request.__init__(url, callback=None, context=None, method='GET', body=None, headers=None, cookies=None, url_encoding='utf-8', dont_filter=None)
 
     Instantiates a ``Request`` object with the given arguments:
 
@@ -50,7 +73,7 @@ Methods
     ``callback`` is a function that will be called with the response of this
     request (once its downloaded) as its first parameter
 
-    ``context`` can be a dict which will be accesible in the callback function
+    ``context`` can be a dict which will be accessible in the callback function
     in ``response.request.context`` in the callback function
 
     ``method`` is a string with the HTTP method of this request
@@ -66,10 +89,6 @@ Methods
     The request URL will be percent encoded using this encoding before
     downloading 
 
-    ``link_text`` is a string describing the URL of this requests. For example,
-    in ``<a href="http://www.example.com/">Example site</a>`` the ``link_text``
-    would be ``"Example site"``
-
     ``dont_filter`` is a boolean which indicates that this request should not
     be filtered by the scheduler. This is used when you want to perform an
     identical request multiple times, for whatever reason
@@ -89,7 +108,19 @@ Attributes
 
 .. attribute:: Response.headers
 
-    A dict of the response headers.
+    A dictionary-like object which contains the response headers.
+
+.. attribute:: Response.meta
+
+    A dict that contains arbitrary metadata fro this response. It works like
+    :attr:`Request.meta` for Request objects. See that attribute help for more
+    info.
+
+.. attribute:: Response.cache
+
+    A dict that contains arbitrary cached data for this response. It works like
+    :attr:`Request.cache` for Request objects. See that attribute help for more
+    info.
 
 Methods
 -------

scrapy/trunk/scrapy/contrib/response/soup.py

@@ -16,7 +16,7 @@ class ResponseSoup(object):
 
 def getsoup(response, **kwargs):
     # TODO: use different cache buckets depending on constructor parameters
-    if not hasattr(response, '_soup'):
+    if 'soup' not in response.cache:
         body = response.body.to_string() if response.body is not None else ""
-        setattr(response, '_soup', BeautifulSoup(body, **kwargs))
-    return response._soup
+        response.cache['soup'] = BeautifulSoup(body, **kwargs)
+    return response.cache['soup']

scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py

@@ -20,17 +20,19 @@ class DepthMiddleware(object):
     def process_spider_output(self, response, result, spider):
         def _filter(request):
             if isinstance(request, Request):
-                request.depth = response.request.depth + 1
-                if self.maxdepth and request.depth > self.maxdepth:
+                depth = response.request.meta['depth'] + 1
+                request.meta['depth'] = depth
+                if self.maxdepth and depth > self.maxdepth:
                     log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name)
                     return False
                 elif self.stats:
-                    stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, request.depth))
-                    if request.depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0):
-                        stats.setpath('%s/request_depth_max' % spider.domain_name, request.depth)
+                    stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, depth))
+                    if depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0):
+                        stats.setpath('%s/request_depth_max' % spider.domain_name, depth)
             return True
 
-        if self.stats and response.request.depth == 0: # otherwise we loose stats for depth=0 
+        if self.stats and 'depth' not in response.request.meta: # otherwise we loose stats for depth=0 
+            response.request.meta['depth'] = 0
             stats.incpath('%s/request_depth_count/0' % spider.domain_name)
 
         return (r for r in result or () if _filter(r))

scrapy/trunk/scrapy/contrib/spiders/crawl.py

@@ -90,7 +90,8 @@ class CrawlSpider(BaseSpider):
                 links = rule.process_links(links)
             seen = seen.union(links)
             for link in links:
-                r = Request(url=link.url, link_text=link.text)
+                r = Request(url=link.url)
+                r.meta['link_text'] = link.text
                 r.append_callback(self._response_downloaded, rule.callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow)
                 requests.append(r)
         return requests

scrapy/trunk/scrapy/http/request.py

@@ -1,5 +1,12 @@
+"""
+This module implements the Request class which is used to represent HTTP
+requests in Scrapy.
+
+See documentation in docs/ref/request-response.rst
+"""
+
 import urllib
-from copy import copy
+import copy
 
 from twisted.internet import defer
 
@@ -12,7 +19,7 @@ class Request(object):
 
     def __init__(self, url, callback=None, context=None, method='GET',
         body=None, headers=None, cookies=None,
-        url_encoding='utf-8', link_text='', dont_filter=None, domain=None):
+        url_encoding='utf-8', dont_filter=None, domain=None):
 
         self.encoding = url_encoding  # this one has to be set first
         self.set_url(url)
@@ -37,13 +44,11 @@ class Request(object):
         self.context = context or {}
         # dont_filter be filtered by scheduler
         self.dont_filter = dont_filter
-        self.depth = 0
-        self.link_text = link_text
         #allows to directly specify the spider for the request
         self.domain = domain
 
-        # bucket to store cached data such as fingerprint and others
-        self._cache = {}
+        self.meta = {}
+        self.cache = {}
         
     def append_callback(self, callback, *args, **kwargs):
         if isinstance(callback, defer.Deferred):
@@ -79,12 +84,12 @@ class Request(object):
 
     def copy(self):
         """Clone request except `context` attribute"""
-        new = copy(self)
-        new._cache = {}
+        new = copy.copy(self)
+        new.cache = {}
         for att in self.__dict__:
-            if att not in ['_cache', 'context', 'url', 'deferred']:
+            if att not in ['cache', 'context', 'url', 'deferred']:
                 value = getattr(self, att)
-                setattr(new, att, copy(value))
+                setattr(new, att, copy.copy(value))
         new.deferred = defer.Deferred()
         new.context = self.context # requests shares same context dictionary
         return new

scrapy/trunk/scrapy/http/response.py

@@ -1,3 +1,10 @@
+"""
+This module implements the Response class which is used to represent HTTP
+responses in Scrapy.
+
+See documentation in docs/ref/request-response.rst
+"""
+
 import re
 import hashlib
 import copy
@@ -34,6 +41,8 @@ class Response(object) :
         self.body = ResponseBody(body, self.headers_encoding())
         self.cached = False
         self.request = None # request which originated this response
+        self.meta = {}
+        self.cache = {}
 
     def version(self):
         """A hash of the contents of this response"""
@@ -85,6 +94,7 @@ class Response(object) :
         # Response.__init__ forbids the use of ResponseBody instances
         if 'body' not in kw:
             newresp.body = samebody()
+        newresp.meta = self.meta.copy()
         return newresp
 
     def to_string(self):

scrapy/trunk/scrapy/tests/test_http_request.py

@@ -79,5 +79,20 @@ class RequestTest(unittest.TestCase):
         self.assert_(isinstance(r.url, str))
         self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
 
+    def test_copy(self):
+        """Test Request copy"""
+        
+        r1 = Request("http://www.example.com")
+        r1.meta['foo'] = 'bar'
+        r1.cache['lala'] = 'lolo'
+        r2 = r1.copy()
+
+        assert r1.cache
+        assert not r2.cache
+
+        # make sure meta dict is shallow copied
+        assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
+        self.assertEqual(r1.meta, r2.meta)
+
 if __name__ == "__main__":
     unittest.main()

scrapy/trunk/scrapy/tests/test_http_response.py

@@ -1,7 +1,7 @@
-from unittest import TestCase, main
+import unittest
 from scrapy.http import Response, ResponseBody
 
-class ResponseTest(TestCase):
+class ResponseTest(unittest.TestCase):
     def test_init(self):
         # Response requires domain and url
         self.assertRaises(Exception, Response)
@@ -14,7 +14,22 @@ class ResponseTest(TestCase):
         # test presence of all optional parameters
         self.assertTrue(isinstance(Response('example.com', 'http://example.com/', original_url='http://example.com/None', headers={}, status=200, body=None), Response))
 
-class ResponseBodyTest(TestCase):
+    def test_copy(self):
+        """Test Response copy"""
+        
+        r1 = Response('example.com', "http://www.example.com")
+        r1.meta['foo'] = 'bar'
+        r1.cache['lala'] = 'lolo'
+        r2 = r1.copy()
+
+        assert r1.cache
+        assert not r2.cache
+
+        # make sure meta dict is shallow copied
+        assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
+        self.assertEqual(r1.meta, r2.meta)
+
+class ResponseBodyTest(unittest.TestCase):
     unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'
 
     def test_encoding(self):
@@ -34,4 +49,4 @@ class ResponseBodyTest(TestCase):
         self.assertEqual(cp1251_body.to_string('utf-8'), self.unicode_string.encode('utf-8'))
 
 if __name__ == "__main__":
-    main()
+    unittest.main()

scrapy/trunk/scrapy/tests/test_utils_request.py

@@ -15,7 +15,7 @@ class UtilsRequestTest(unittest.TestCase):
         self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
 
         # make sure caching is working
-        self.assertEqual(request_fingerprint(r1), r1._cache['fingerprint'])
+        self.assertEqual(request_fingerprint(r1), r1.cache['fingerprint'])
 
         r1 = Request("http://www.example.com/members/offers.html")
         r2 = Request("http://www.example.com/members/offers.html")

scrapy/trunk/scrapy/utils/request.py

@@ -43,7 +43,7 @@ def request_fingerprint(request, include_headers=()):
         cachekey = 'fingerprint'
 
     try:
-        return request._cache[cachekey]
+        return request.cache[cachekey]
     except KeyError:
         fp = hashlib.sha1()
         fp.update(request.method)
@@ -54,7 +54,7 @@ def request_fingerprint(request, include_headers=()):
                 fp.update(hdr)
                 fp.update(request.headers.get(hdr, ''))
         fphash = fp.hexdigest()
-        request._cache[cachekey] = fphash
+        request.cache[cachekey] = fphash
         return fphash
 
 def request_authenticate(request, username, password):

scrapy/trunk/scrapy/xpath/extension.py

@@ -12,9 +12,9 @@ class ResponseLibxml2(object):
         setattr(Response, 'getlibxml2doc', getlibxml2doc)
 
 def getlibxml2doc(response, constructor=xmlDoc_from_html):
-    attr = '_lx2doc_%s' % constructor.__name__
-    if not hasattr(response, attr):
+    cachekey = 'lx2doc_%s' % constructor.__name__
+    if cachekey not in response.cache:
         lx2doc = Libxml2Document(response, constructor=constructor)
-        setattr(response, attr, lx2doc)
-    return getattr(response, attr)
+        response.cache[cachekey] = lx2doc
+    return response.cache[cachekey]