diff --git a/docs/ref/request-response.rst b/docs/ref/request-response.rst index 8150d6dee..a94674c3b 100644 --- a/docs/ref/request-response.rst +++ b/docs/ref/request-response.rst @@ -156,10 +156,6 @@ Request objects in the ``meta`` argument). The :attr:`Request.cache` attribute is always cleared. See also :ref:`ref-request-callback-arguments`. - .. method:: Request.httprepr() - - Return a string with the raw HTTP representation of this response. - .. _ref-request-callback-copy: Caveats with copying Requests and callbacks @@ -436,10 +432,6 @@ Response objects is given in the ``meta`` argument). The :attr:`Response.cache` attribute is always cleared. - .. method:: Response.httprepr() - - Return a string with the raw HTTP representation of this response. - .. _ref-response-subclasses: Response subclasses diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py index d8c337848..0a4631696 100644 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ b/scrapy/contrib/downloadermiddleware/stats.py @@ -1,4 +1,6 @@ from scrapy.core.exceptions import NotConfigured +from scrapy.utils.request import request_httprepr +from scrapy.utils.response import response_httprepr from scrapy.stats import stats from scrapy.conf import settings @@ -18,7 +20,7 @@ class DownloaderStats(object): stats.inc_value('downloader/request_count') stats.inc_value('downloader/request_count', domain=domain) stats.inc_value('downloader/request_method_count/%s' % request.method, domain=domain) - reqlen = len(request.httprepr()) + reqlen = len(request_httprepr(request)) stats.inc_value('downloader/request_bytes', reqlen, domain=domain) stats.inc_value('downloader/request_bytes', reqlen) @@ -27,7 +29,7 @@ class DownloaderStats(object): stats.inc_value('downloader/response_count') stats.inc_value('downloader/response_count', domain=domain) stats.inc_value('downloader/response_status_count/%s' % response.status, domain=domain) - reslen = len(response.httprepr()) + reslen = len(response_httprepr(response)) stats.inc_value('downloader/response_bytes', reslen, domain=domain) stats.inc_value('downloader/response_bytes', reslen) return response diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index a38ef3d78..a27610de1 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -30,6 +30,7 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured from scrapy.core import signals +from scrapy.utils.response import response_httprepr from scrapy.stats import stats from scrapy.http import Request from scrapy import log @@ -86,7 +87,7 @@ class ItemSamplerMiddleware(object): def process_spider_input(self, response, spider): if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_domain: return [] - elif max_response_size and max_response_size > len(response.httprepr()): + elif max_response_size and max_response_size > len(response_httprepr(response)): return [] def process_spider_output(self, response, result, spider): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 2556b7bb3..4279b7b98 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -96,18 +96,3 @@ class Request(object): meta=self.meta if meta is None else meta, encoding=self.encoding if encoding is None else encoding, dont_filter=self.dont_filter if dont_filter is None else dont_filter) - - def httprepr(self): - """ Return raw HTTP request representation (as string). This is - provided only for reference since it's not the actual stream of bytes - that will be send when performing the request (that's controlled by - Twisted). - """ - - s = "%s %s HTTP/1.1\r\n" % (self.method, self.url) - s += "Host: %s\r\n" % self.url.hostname - if self.headers: - s += self.headers.to_string() + "\r\n" - s += "\r\n" - s += self.body - return s diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 830a8dff7..851c99f10 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,8 +7,6 @@ See documentation in docs/ref/request-response.rst import copy -from twisted.web.http import RESPONSES - from scrapy.http.url import Url from scrapy.http.headers import Headers @@ -63,17 +61,3 @@ class Response(object): flags=self.flags if flags is None else flags, **kwargs) return new - - def httprepr(self): - """ - Return raw HTTP response representation (as string). This is provided - only for reference, since it's not the exact stream of bytes that was - received (that's not exposed by Twisted). - """ - - s = "HTTP/1.1 %d %s\r\n" % (self.status, RESPONSES[self.status]) - if self.headers: - s += self.headers.to_string() + "\r\n" - s += "\r\n" - s += self.body - return s diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 3c50ffec9..d1d31cd33 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -166,13 +166,6 @@ class RequestTest(unittest.TestCase): self.assertEqual(r4.meta, {}) assert r4.dont_filter is False - def test_httprepr(self): - r1 = Request("http://www.example.com") - self.assertEqual(r1.httprepr(), 'GET http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n') - - r1 = Request("http://www.example.com", method='POST', headers={"Content-type": "text/html"}, body="Some body") - self.assertEqual(r1.httprepr(), 'POST http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body') - class FormRequestTest(unittest.TestCase): diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index ac89c5ae6..c5803e8f2 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -104,13 +104,6 @@ class ResponseTest(unittest.TestCase): self.assertEqual(r4.meta, {}) self.assertEqual(r4.flags, []) - def test_httprepr(self): - r1 = Response("http://www.example.com") - self.assertEqual(r1.httprepr(), 'HTTP/1.1 200 OK\r\n\r\n') - - r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body="Some body") - self.assertEqual(r1.httprepr(), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') - def test_encoding(self): unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442' self.assertRaises(TypeError, Response, 'http://www.example.com', body=u'unicode body') diff --git a/scrapy/tests/test_utils_request.py b/scrapy/tests/test_utils_request.py index 2a61d06ca..5b0f069d9 100644 --- a/scrapy/tests/test_utils_request.py +++ b/scrapy/tests/test_utils_request.py @@ -1,14 +1,10 @@ import unittest from scrapy.http import Request -from scrapy.utils.request import request_fingerprint, request_authenticate +from scrapy.utils.request import request_fingerprint, request_authenticate, request_httprepr class UtilsRequestTest(unittest.TestCase): def test_request_fingerprint(self): - url = 'http://www.scrapy.org' - r = Request(url=url) - urlhash = request_fingerprint(r) - r1 = Request("http://www.example.com/query?id=111&cat=222") r2 = Request("http://www.example.com/query?cat=222&id=111") self.assertEqual(request_fingerprint(r1), request_fingerprint(r1)) @@ -59,11 +55,17 @@ class UtilsRequestTest(unittest.TestCase): fp2 = request_fingerprint(r2) self.assertNotEqual(fp1, fp2) - def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, 'someuser', 'somepass') self.assertEqual(r.headers['Authorization'], 'Basic c29tZXVzZXI6c29tZXBhc3M=') + def test_request_httprepr(self): + r1 = Request("http://www.example.com") + self.assertEqual(request_httprepr(r1), 'GET http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n') + + r1 = Request("http://www.example.com", method='POST', headers={"Content-type": "text/html"}, body="Some body") + self.assertEqual(request_httprepr(r1), 'POST http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body') + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 986488670..34a78e392 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -1,6 +1,6 @@ import unittest from scrapy.http import Response, TextResponse -from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh +from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh, response_httprepr class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body='dummy_response') @@ -33,7 +33,7 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(get_base_url(response), 'http://example.org/something') def test_get_meta_refresh(self): - body=""" + body = """ Dummy blahablsdfsal& @@ -42,14 +42,22 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) # refresh without url should return (None, None) - body="""""" + body = """""" response = Response(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (None, None)) - body="""""" response = Response(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + def test_response_httprepr(self): + r1 = Response("http://www.example.com") + self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n') + + r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body="Some body") + self.assertEqual(response_httprepr(r1), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 212503765..7780ecc44 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -72,3 +72,17 @@ def request_info(request): fp = request_fingerprint(request) return "" % (request.method, request.url, fp[:8]) +def request_httprepr(request): + """Return the raw HTTP representation (as string) of the given request. + This is provided only for reference since it's not the actual stream of + bytes that will be send when performing the request (that's controlled + by Twisted). + """ + + s = "%s %s HTTP/1.1\r\n" % (request.method, request.url) + s += "Host: %s\r\n" % request.url.hostname + if request.headers: + s += request.headers.to_string() + "\r\n" + s += "\r\n" + s += request.body + return s diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 596d96807..c7bf82ec0 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -4,7 +4,10 @@ scrapy.http.Response objects """ import re + from twisted.web import http +from twisted.web.http import RESPONSES + from scrapy.http.response import Response def body_or_str(obj, unicode=True): @@ -48,3 +51,16 @@ def response_status_message(status): 404 Not Found """ return '%s %s' % (status, http.responses.get(int(status))) + +def response_httprepr(response): + """Return raw HTTP representation (as string) of the given response. This + is provided only for reference, since it's not the exact stream of bytes + that was received (that's not exposed by Twisted). + """ + + s = "HTTP/1.1 %d %s\r\n" % (response.status, RESPONSES[response.status]) + if response.headers: + s += response.headers.to_string() + "\r\n" + s += "\r\n" + s += response.body + return s