From 654b49c86e79d571190e60ba28c8c551582bba28 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 17 Jan 2009 23:57:53 +0000 Subject: [PATCH] added meta argument to Request & Response constructors --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40741 --- scrapy/trunk/docs/ref/request-response.rst | 8 ++++++- scrapy/trunk/scrapy/http/request.py | 5 +++-- scrapy/trunk/scrapy/http/response.py | 4 ++-- .../trunk/scrapy/tests/test_http_request.py | 22 ++++++++++++++++++- .../trunk/scrapy/tests/test_http_response.py | 21 +++++++++++++++++- 5 files changed, 53 insertions(+), 7 deletions(-) diff --git a/scrapy/trunk/docs/ref/request-response.rst b/scrapy/trunk/docs/ref/request-response.rst index 0a9846d24..6757b4225 100644 --- a/scrapy/trunk/docs/ref/request-response.rst +++ b/scrapy/trunk/docs/ref/request-response.rst @@ -21,7 +21,7 @@ generated the request. Request objects =============== -.. class:: Request(url, callback=None, method='GET', body=None, headers=None, cookies=None, url_encoding='utf-8', dont_filter=None) +.. class:: Request(url, callback=None, method='GET', body=None, headers=None, cookies=None, meta=None, url_encoding='utf-8', dont_filter=None) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -34,6 +34,9 @@ Request objects ``method`` is a string with the HTTP method of this request + ``meta`` is a dict containing the initial values for the + :attr:`Request.meta` attribute. If passed, the dict will be shallow copied. + ``body`` is a string containing the request body or None if the request doesn't contain a body (ex. GET requests) @@ -130,6 +133,9 @@ Response objects ``body`` is a string (or unicode) containing the response body + ``meta`` is a dict containing the initial values for the + :attr:`Response.meta` attribute. If passed, the dict will be shallow copied. + Attributes ---------- diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py index c0982c11c..ff5e21c8f 100644 --- a/scrapy/trunk/scrapy/http/request.py +++ b/scrapy/trunk/scrapy/http/request.py @@ -18,7 +18,7 @@ from scrapy.utils.defer import chain_deferred class Request(object): def __init__(self, url, callback=None, method='GET', - body=None, headers=None, cookies=None, + body=None, headers=None, cookies=None, meta=None, url_encoding='utf-8', dont_filter=None, domain=None): self.encoding = url_encoding # this one has to be set first @@ -45,7 +45,7 @@ class Request(object): #allows to directly specify the spider for the request self.domain = domain - self.meta = {} + self.meta = {} if meta is None else dict(meta) self.cache = {} def append_callback(self, callback, *args, **kwargs): @@ -73,6 +73,7 @@ class Request(object): 'headers': self.headers, 'cookies': self.cookies, 'body': self.body, + 'meta': self.meta, } return "%s(%s)" % (self.__class__.__name__, repr(d)) diff --git a/scrapy/trunk/scrapy/http/response.py b/scrapy/trunk/scrapy/http/response.py index a5033030e..3ea480611 100644 --- a/scrapy/trunk/scrapy/http/response.py +++ b/scrapy/trunk/scrapy/http/response.py @@ -19,7 +19,7 @@ class Response(object): _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) - def __init__(self, domain, url, status=200, headers=None, body=None): + def __init__(self, domain, url, status=200, headers=None, body=None, meta=None): self.domain = domain self.url = Url(url) self.headers = Headers(headers or {}) @@ -32,7 +32,7 @@ class Response(object): self.body = None self.cached = False self.request = None - self.meta = {} + self.meta = {} if meta is None else dict(meta) self.cache = {} def headers_encoding(self): diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py index 9f336f632..cee41cb15 100644 --- a/scrapy/trunk/scrapy/tests/test_http_request.py +++ b/scrapy/trunk/scrapy/tests/test_http_request.py @@ -1,9 +1,29 @@ import unittest -from scrapy.http import Request, Headers +from scrapy.http import Request, Headers, Url from scrapy.core.scheduler import GroupFilter class RequestTest(unittest.TestCase): + def test_init(self): + r = Request("http://www.example.com") + assert isinstance(r.url, Url) + self.assertEqual(r.url, "http://www.example.com") + self.assertEqual(r.method, "GET") + + assert isinstance(r.headers, Headers) + self.assertEqual(r.headers, {}) + self.assertEqual(r.meta, {}) + + meta = {"lala": "lolo"} + headers = {"caca": "coco"} + body = "a body" + r = Request("http://www.example.com", meta=meta, headers=headers, body="a body") + + assert r.meta is not meta + self.assertEqual(r.meta, meta) + assert r.headers is not headers + self.assertEqual(r.headers["caca"], "coco") + def test_groupfilter(self): k1 = "id1" k2 = "id1" diff --git a/scrapy/trunk/scrapy/tests/test_http_response.py b/scrapy/trunk/scrapy/tests/test_http_response.py index 7224d9102..117bd8ab0 100644 --- a/scrapy/trunk/scrapy/tests/test_http_response.py +++ b/scrapy/trunk/scrapy/tests/test_http_response.py @@ -1,5 +1,5 @@ import unittest -from scrapy.http import Response +from scrapy.http import Response, Headers, Url from scrapy.http.response import _ResponseBody class ResponseTest(unittest.TestCase): @@ -15,6 +15,25 @@ class ResponseTest(unittest.TestCase): # test presence of all optional parameters self.assertTrue(isinstance(Response('example.com', 'http://example.com/', headers={}, status=200, body=None), Response)) + r = Response("domain.com", "http://www.example.com") + assert isinstance(r.url, Url) + self.assertEqual(r.url, "http://www.example.com") + self.assertEqual(r.status, 200) + + assert isinstance(r.headers, Headers) + self.assertEqual(r.headers, {}) + self.assertEqual(r.meta, {}) + + meta = {"lala": "lolo"} + headers = {"caca": "coco"} + body = "a body" + r = Response("example.com", "http://www.example.com", meta=meta, headers=headers, body="a body") + + assert r.meta is not meta + self.assertEqual(r.meta, meta) + assert r.headers is not headers + self.assertEqual(r.headers["caca"], "coco") + def test_copy(self): """Test Response copy"""