From b1745f49f1b2cfd3078508a30a6f5c55d1f50603 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 17 Jan 2009 15:57:28 +0000 Subject: [PATCH] removed deprecated original_url attribute from Response objects (it can be accessed through Response.request.url) --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40736 --- .../contrib/downloadermiddleware/cache.py | 4 +--- .../trunk/scrapy/core/downloader/handlers.py | 2 +- scrapy/trunk/scrapy/http/response.py | 21 +++++-------------- .../trunk/scrapy/tests/test_http_response.py | 2 +- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py index 2f401a0cf..ea69fa876 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py @@ -150,11 +150,10 @@ class Cache(object): responseheaders = f.read() url = metadata['url'] - original_url = metadata.get('original_url', url) headers = Headers(responseheaders) status = metadata['status'] - response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody) + response = Response(domain=domain, url=url, headers=headers, status=status, body=responsebody) response.cached = True return response @@ -168,7 +167,6 @@ class Cache(object): 'method': request.method, 'status': response.status, 'domain': response.domain, - 'original_url': response.original_url, 'timestamp': datetime.datetime.utcnow(), } diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/trunk/scrapy/core/downloader/handlers.py index 3ef4f48ed..0a8fd3d2a 100644 --- a/scrapy/trunk/scrapy/core/downloader/handlers.py +++ b/scrapy/trunk/scrapy/core/downloader/handlers.py @@ -49,7 +49,7 @@ def download_http(request, spider): body = body or '' status = factory.status headers = Headers(factory.response_headers) - r = Response(domain=spider.domain_name, url=request.url, headers=headers, status=status, body=body) + r = Response(domain=spider.domain_name, url=request.url, status=status, headers=headers, body=body) signals.send_catch_log(signal=signals.request_uploaded, sender='download_http', request=request, spider=spider) signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider) return r diff --git a/scrapy/trunk/scrapy/http/response.py b/scrapy/trunk/scrapy/http/response.py index 9566d88b5..589a33935 100644 --- a/scrapy/trunk/scrapy/http/response.py +++ b/scrapy/trunk/scrapy/http/response.py @@ -1,6 +1,6 @@ """ This module implements the Response class which is used to represent HTTP -responses in Scrapy. +esponses in Scrapy. See documentation in docs/ref/request-response.rst """ @@ -17,23 +17,13 @@ from scrapy.http.headers import Headers from twisted.web import http reason_phrases = http.RESPONSES -class Response(object) : - """HTTP responses +class Response(object): - Arguments: - * Domain - the spider domain for the page - * url - the final url for the resource - * original_url - the url requested - * headers - HTTP headers - * status - HTTP status code - * body - Body object containing the content of the response - """ _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) - def __init__(self, domain, url, original_url=None, headers=None, status=200, body=None): + def __init__(self, domain, url, status=200, headers=None, body=None): self.domain = domain self.url = Url(url) - self.original_url = Url(original_url) if original_url else url # different if redirected or escaped self.headers = Headers(headers or {}) self.status = status # ResponseBody is not meant to be used directly (use .replace instead) @@ -58,8 +48,8 @@ class Response(object) : return encoding.group(1) def __repr__(self): - return "Response(domain=%s, url=%s, original_url=%s, headers=%s, status=%s, body=%s)" % \ - (repr(self.domain), repr(self.url), repr(self.original_url), repr(self.headers), repr(self.status), repr(self.body)) + return "Response(domain=%s, url=%s, headers=%s, status=%s, body=%s)" % \ + (repr(self.domain), repr(self.url), repr(self.headers), repr(self.status), repr(self.body)) def __str__(self): version = '%s..%s' % (self.version()[:4], self.version()[-4:]) @@ -87,7 +77,6 @@ class Response(object) : return copy.deepcopy(self.body) newresp = Response(kw.get('domain', self.domain), kw.get('url', self.url), - original_url=kw.get('original_url', self.original_url), headers=kw.get('headers', sameheaders()), status=kw.get('status', self.status), body=kw.get('body')) diff --git a/scrapy/trunk/scrapy/tests/test_http_response.py b/scrapy/trunk/scrapy/tests/test_http_response.py index 9bf41ce78..4fd165453 100644 --- a/scrapy/trunk/scrapy/tests/test_http_response.py +++ b/scrapy/trunk/scrapy/tests/test_http_response.py @@ -12,7 +12,7 @@ class ResponseTest(unittest.TestCase): self.assertTrue(isinstance(Response('example.com', 'http://example.com/', body='body'), Response)) self.assertRaises(AssertionError, Response, 'example.com', 'http://example.com/', body=ResponseBody('body', 'utf-8')) # test presence of all optional parameters - self.assertTrue(isinstance(Response('example.com', 'http://example.com/', original_url='http://example.com/None', headers={}, status=200, body=None), Response)) + self.assertTrue(isinstance(Response('example.com', 'http://example.com/', headers={}, status=200, body=None), Response)) def test_copy(self): """Test Response copy"""