From 0c9f7257a211490bc63a0a0cf1e4e2db86521d7d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 18 Jan 2009 17:52:21 +0000 Subject: [PATCH] added Request.replace method, improved tests for replace/copy method in Request/Response classes --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40745 --- scrapy/trunk/scrapy/http/request.py | 20 ++++++++++++---- scrapy/trunk/scrapy/http/response.py | 16 ++++++------- .../trunk/scrapy/tests/test_http_request.py | 23 ++++++++++++++++++- .../trunk/scrapy/tests/test_http_response.py | 22 +++++++++++++++++- 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py index 876d056fa..07b517e1f 100644 --- a/scrapy/trunk/scrapy/http/request.py +++ b/scrapy/trunk/scrapy/http/request.py @@ -17,9 +17,8 @@ from scrapy.utils.defer import chain_deferred class Request(object): - def __init__(self, url, callback=None, method='GET', - body=None, headers=None, cookies=None, meta=None, - url_encoding='utf-8', dont_filter=None): + def __init__(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, url_encoding='utf-8', dont_filter=None): self.encoding = url_encoding # this one has to be set first self.set_url(url) @@ -69,14 +68,14 @@ class Request(object): 'method': self.method, 'url': self.url, 'headers': self.headers, - 'cookies': self.cookies, 'body': self.body, + 'cookies': self.cookies, 'meta': self.meta, } return "%s(%s)" % (self.__class__.__name__, repr(d)) def copy(self): - """Return a new request cloned from this one""" + """Return a copy a of this Request""" new = copy.copy(self) new.cache = {} for att in self.__dict__: @@ -86,6 +85,17 @@ class Request(object): new.deferred = defer.Deferred() return new + def replace(self, url=None, method=None, headers=None, body=None, cookies=None, meta=None): + """Create a new Request with the same attributes except for those + given new values. + """ + return self.__class__(url=url or self.url, + method=method or self.method, + headers=headers or copy.deepcopy(self.headers), + body=body or self.body, + cookies=cookies or self.cookies, + meta=meta or self.meta) + def httprepr(self): """ Return raw HTTP request representation (as string). This is provided only for reference since it's not the actual stream of bytes diff --git a/scrapy/trunk/scrapy/http/response.py b/scrapy/trunk/scrapy/http/response.py index 38318caff..7896c001d 100644 --- a/scrapy/trunk/scrapy/http/response.py +++ b/scrapy/trunk/scrapy/http/response.py @@ -51,24 +51,21 @@ class Response(object): return "<%d %s>" % (self.status, self.url) def copy(self): - """Create a new Response based on the current one""" + """Return a copy of this Response""" return self.replace() - def replace(self, url=None, status=None, headers=None, body=None): + def replace(self, url=None, status=None, headers=None, body=None, meta=None): """Create a new Response with the same attributes except for those given new values. - - Example: - - >>> newresp = oldresp.replace(body="New body") """ new = self.__class__(url=url or self.url, status=status or self.status, headers=headers or copy.deepcopy(self.headers), - body=body) + meta=meta or self.meta) if body is None: new.body = copy.deepcopy(self.body) - new.meta = self.meta.copy() + else: + new.body = _ResponseBody(body, self.headers_encoding()) return new def httprepr(self): @@ -184,3 +181,6 @@ class _ResponseBody(object): def __len__(self): return len(self._content) + def __eq__(self, other): + return self._content == other._content and self.declared_encoding == other.declared_encoding + diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py index cda8423a1..c5721c197 100644 --- a/scrapy/trunk/scrapy/tests/test_http_request.py +++ b/scrapy/trunk/scrapy/tests/test_http_request.py @@ -105,7 +105,10 @@ class RequestTest(unittest.TestCase): def test_copy(self): """Test Request copy""" - r1 = Request("http://www.example.com") + def somecallback(): + pass + + r1 = Request("http://www.example.com", callback=somecallback) r1.meta['foo'] = 'bar' r1.cache['lala'] = 'lolo' r2 = r1.copy() @@ -113,10 +116,18 @@ class RequestTest(unittest.TestCase): assert r1.cache assert not r2.cache + assert r1.deferred is not r2.deferred + # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" self.assertEqual(r1.meta, r2.meta) + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical" + self.assertEqual(r1.headers, r2.headers) + + # Request.body can be identical since it's an immutable object (str) + def test_copy_inherited_classes(self): """Test Request children copies preserve their class""" @@ -128,6 +139,16 @@ class RequestTest(unittest.TestCase): assert type(r2) is CustomRequest + def test_replace(self): + """Test Request.replace() method""" + hdrs = Headers({"key": "value"}) + r1 = Request("http://www.example.com") + r2 = r1.replace(method="POST", body="New body", headers=hdrs) + self.assertEqual(r1.url, r2.url) + self.assertEqual((r1.method, r2.method), ("GET", "POST")) + self.assertEqual((r1.body, r2.body), (None, "New body")) + self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) + def test_httprepr(self): r1 = Request("http://www.example.com") self.assertEqual(r1.httprepr(), 'GET http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n') diff --git a/scrapy/trunk/scrapy/tests/test_http_response.py b/scrapy/trunk/scrapy/tests/test_http_response.py index 4877f1ea9..76c3f1d4e 100644 --- a/scrapy/trunk/scrapy/tests/test_http_response.py +++ b/scrapy/trunk/scrapy/tests/test_http_response.py @@ -36,7 +36,7 @@ class ResponseTest(unittest.TestCase): def test_copy(self): """Test Response copy""" - r1 = Response("http://www.example.com") + r1 = Response("http://www.example.com", body="Some body") r1.meta['foo'] = 'bar' r1.cache['lala'] = 'lolo' r2 = r1.copy() @@ -48,6 +48,14 @@ class ResponseTest(unittest.TestCase): assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" self.assertEqual(r1.meta, r2.meta) + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical" + self.assertEqual(r1.headers, r2.headers) + + # make sure body is shallow copied + assert r1.body is not r2.body, "body must be a shallow copy, not identical" + self.assertEqual(r1.body, r2.body) + def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" @@ -59,6 +67,18 @@ class ResponseTest(unittest.TestCase): assert type(r2) is CustomResponse + def test_replace(self): + """Test Response.replace() method""" + hdrs = Headers({"key": "value"}) + r1 = Response("http://www.example.com") + r2 = r1.replace(status=301, body="New body", headers=hdrs) + assert r1.body is None + assert isinstance(r2.body, _ResponseBody) + self.assertEqual(r1.url, r2.url) + self.assertEqual((r1.status, r2.status), (200, 301)) + self.assertEqual((r1.body, r2.body.to_string()), (None, "New body")) + self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) + def test_httprepr(self): r1 = Response("http://www.example.com") self.assertEqual(r1.httprepr(), 'HTTP/1.1 200 OK\r\n\r\n')