From 7e640da4332180477534fb242a53a5fedabb133d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 17 Jan 2009 22:11:54 +0000 Subject: [PATCH] renamed to_string() Request and Response methods to httprepr(). removed __len__() from Request and Response --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40739 --- scrapy/trunk/docs/ref/request-response.rst | 6 +++--- scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py | 4 ++-- scrapy/trunk/scrapy/contrib/itemsampler.py | 2 +- scrapy/trunk/scrapy/http/request.py | 6 +----- scrapy/trunk/scrapy/http/response.py | 6 +----- scrapy/trunk/scrapy/tests/test_http_request.py | 6 +++--- scrapy/trunk/scrapy/tests/test_http_response.py | 6 +++--- 7 files changed, 14 insertions(+), 22 deletions(-) diff --git a/scrapy/trunk/docs/ref/request-response.rst b/scrapy/trunk/docs/ref/request-response.rst index 2cc033e2a..49806c3a5 100644 --- a/scrapy/trunk/docs/ref/request-response.rst +++ b/scrapy/trunk/docs/ref/request-response.rst @@ -110,7 +110,7 @@ Methods given new values by whichever keyword arguments are specified. The attribute :attr:`Request.meta` is copied, while :attr:`Request.cache` is not. -.. method:: Reponse.to_string() +.. method:: Request.httprepr() Return a string with the raw HTTP representation of this response. @@ -139,7 +139,7 @@ Attributes .. attribute:: Response.url - A string containing the URL of the reponse. + A string containing the URL of the response. .. attribute:: Response.status @@ -198,6 +198,6 @@ Methods given new values by whichever keyword arguments are specified. The attribute :attr:`Response.meta` is copied, while :attr:`Response.cache` is not. -.. method:: Reponse.to_string() +.. method:: Response.httprepr() Return a string with the raw HTTP representation of this response. diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py index 9598cc463..5ae74bfd3 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py @@ -23,7 +23,7 @@ class DownloaderStats(object): stats.incpath('_global/downloader/request_count') stats.incpath('%s/downloader/request_count' % spider.domain_name) stats.incpath('%s/downloader/request_method_count/%s' % (spider.domain_name, request.method)) - reqlen = len(request) + reqlen = len(request.httprepr()) stats.incpath('%s/downloader/request_bytes' % spider.domain_name, reqlen) stats.incpath('_global/downloader/request_bytes', reqlen) @@ -43,6 +43,6 @@ class DownloaderStats(object): stats.incpath('_global/downloader/response_count') stats.incpath('%s/downloader/response_count' % domain) stats.incpath('%s/downloader/response_status_count/%s' % (domain, response.status)) - reslen = len(response) + reslen = len(response.httprepr()) stats.incpath('%s/downloader/response_bytes' % domain, reslen) stats.incpath('_global/downloader/response_bytes', reslen) diff --git a/scrapy/trunk/scrapy/contrib/itemsampler.py b/scrapy/trunk/scrapy/contrib/itemsampler.py index 7f190edc5..d7c1f61ae 100644 --- a/scrapy/trunk/scrapy/contrib/itemsampler.py +++ b/scrapy/trunk/scrapy/contrib/itemsampler.py @@ -86,7 +86,7 @@ class ItemSamplerMiddleware(object): def process_spider_input(self, response, spider): if stats.getpath("%s/items_sampled" % spider.domain_name) >= items_per_domain: return [] - elif max_response_size and max_response_size > len(response): + elif max_response_size and max_response_size > len(response.httprepr()): return [] def process_spider_output(self, response, result, spider): diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py index 6174a0c31..850a4772f 100644 --- a/scrapy/trunk/scrapy/http/request.py +++ b/scrapy/trunk/scrapy/http/request.py @@ -68,10 +68,6 @@ class Request(object): else: return "<%s %s>" % (self.method, self.url) - def __len__(self): - """Return raw HTTP request size""" - return len(self.to_string()) - def __repr__(self): d = { 'method': self.method, @@ -95,7 +91,7 @@ class Request(object): new.context = self.context # requests shares same context dictionary return new - def to_string(self): + def httprepr(self): """ Return raw HTTP request representation (as string). This is provided only for reference since it's not the actual stream of bytes that will be send when performing the request (that's controlled by diff --git a/scrapy/trunk/scrapy/http/response.py b/scrapy/trunk/scrapy/http/response.py index 65fae7414..a5033030e 100644 --- a/scrapy/trunk/scrapy/http/response.py +++ b/scrapy/trunk/scrapy/http/response.py @@ -52,10 +52,6 @@ class Response(object): else: return "<%d %s>" % (self.status, self.url) - def __len__(self): - """Return raw HTTP response size""" - return len(self.to_string()) - def copy(self): """Create a new Response based on the current one""" return self.replace() @@ -78,7 +74,7 @@ class Response(object): new.meta = self.meta.copy() return new - def to_string(self): + def httprepr(self): """ Return raw HTTP response representation (as string). This is provided only for reference, since it's not the exact stream of bytes that was diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py index 3da9da4d1..9f336f632 100644 --- a/scrapy/trunk/scrapy/tests/test_http_request.py +++ b/scrapy/trunk/scrapy/tests/test_http_request.py @@ -105,12 +105,12 @@ class RequestTest(unittest.TestCase): assert type(r2) is CustomRequest - def test_to_string(self): + def test_httprepr(self): r1 = Request("http://www.example.com") - self.assertEqual(r1.to_string(), 'GET http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n') + self.assertEqual(r1.httprepr(), 'GET http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n') r1 = Request("http://www.example.com", method='POST', headers={"Content-type": "text/html"}, body="Some body") - self.assertEqual(r1.to_string(), 'POST http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body\r\n') + self.assertEqual(r1.httprepr(), 'POST http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body\r\n') if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_http_response.py b/scrapy/trunk/scrapy/tests/test_http_response.py index fa7600da7..7224d9102 100644 --- a/scrapy/trunk/scrapy/tests/test_http_response.py +++ b/scrapy/trunk/scrapy/tests/test_http_response.py @@ -41,12 +41,12 @@ class ResponseTest(unittest.TestCase): assert type(r2) is CustomResponse - def test_to_string(self): + def test_httprepr(self): r1 = Response('example.com', "http://www.example.com") - self.assertEqual(r1.to_string(), 'HTTP/1.1 200 OK\r\n\r\n') + self.assertEqual(r1.httprepr(), 'HTTP/1.1 200 OK\r\n\r\n') r1 = Response('example.com', "http://www.example.com", status=404, headers={"Content-type": "text/html"}, body="Some body") - self.assertEqual(r1.to_string(), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body\r\n') + self.assertEqual(r1.httprepr(), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body\r\n') class ResponseBodyTest(unittest.TestCase): unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'