RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

moved httprepr() method (from Request and Response objects) to scrapy.utils functions

This commit is contained in:
Pablo Hoffman 2009-07-25 18:56:12 -03:00
parent bed3c38014
commit 9da66698f3
11 changed files with 56 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/ref/request-response.rst
View File

@ -156,10 +156,6 @@ Request objects
in the ``meta`` argument). The :attr:`Request.cache` attribute is always
cleared. See also :ref:`ref-request-callback-arguments`.
.. method:: Request.httprepr()
Return a string with the raw HTTP representation of this response.
.. _ref-request-callback-copy:
Caveats with copying Requests and callbacks
@ -436,10 +432,6 @@ Response objects
is given in the ``meta`` argument). The :attr:`Response.cache`
attribute is always cleared.
.. method:: Response.httprepr()
Return a string with the raw HTTP representation of this response.
.. _ref-response-subclasses:
Response subclasses

6
scrapy/contrib/downloadermiddleware/stats.py
View File

@ -1,4 +1,6 @@
from scrapy.core.exceptions import NotConfigured
from scrapy.utils.request import request_httprepr
from scrapy.utils.response import response_httprepr
from scrapy.stats import stats
from scrapy.conf import settings
@ -18,7 +20,7 @@ class DownloaderStats(object):
stats.inc_value('downloader/request_count')
stats.inc_value('downloader/request_count', domain=domain)
stats.inc_value('downloader/request_method_count/%s' % request.method, domain=domain)
reqlen = len(request.httprepr())
reqlen = len(request_httprepr(request))
stats.inc_value('downloader/request_bytes', reqlen, domain=domain)
stats.inc_value('downloader/request_bytes', reqlen)
@ -27,7 +29,7 @@ class DownloaderStats(object):
stats.inc_value('downloader/response_count')
stats.inc_value('downloader/response_count', domain=domain)
stats.inc_value('downloader/response_status_count/%s' % response.status, domain=domain)
reslen = len(response.httprepr())
reslen = len(response_httprepr(response))
stats.inc_value('downloader/response_bytes', reslen, domain=domain)
stats.inc_value('downloader/response_bytes', reslen)
return response

3
scrapy/contrib/itemsampler.py
View File

@ -30,6 +30,7 @@ from scrapy.xlib.pydispatch import dispatcher
from scrapy.core.engine import scrapyengine
from scrapy.core.exceptions import NotConfigured
from scrapy.core import signals
from scrapy.utils.response import response_httprepr
from scrapy.stats import stats
from scrapy.http import Request
from scrapy import log
@ -86,7 +87,7 @@ class ItemSamplerMiddleware(object):
def process_spider_input(self, response, spider):
if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_domain:
return []
elif max_response_size and max_response_size > len(response.httprepr()):
elif max_response_size and max_response_size > len(response_httprepr(response)):
return []
def process_spider_output(self, response, result, spider):

15
scrapy/http/request/__init__.py
View File

@ -96,18 +96,3 @@ class Request(object):
meta=self.meta if meta is None else meta,
encoding=self.encoding if encoding is None else encoding,
dont_filter=self.dont_filter if dont_filter is None else dont_filter)
def httprepr(self):
""" Return raw HTTP request representation (as string). This is
provided only for reference since it's not the actual stream of bytes
that will be send when performing the request (that's controlled by
Twisted).
"""
s = "%s %s HTTP/1.1\r\n" % (self.method, self.url)
s += "Host: %s\r\n" % self.url.hostname
if self.headers:
s += self.headers.to_string() + "\r\n"
s += "\r\n"
s += self.body
return s

16
scrapy/http/response/__init__.py
View File

@ -7,8 +7,6 @@ See documentation in docs/ref/request-response.rst
import copy
from twisted.web.http import RESPONSES
from scrapy.http.url import Url
from scrapy.http.headers import Headers
@ -63,17 +61,3 @@ class Response(object):
flags=self.flags if flags is None else flags,
**kwargs)
return new
def httprepr(self):
"""
Return raw HTTP response representation (as string). This is provided
only for reference, since it's not the exact stream of bytes that was
received (that's not exposed by Twisted).
"""
s = "HTTP/1.1 %d %s\r\n" % (self.status, RESPONSES[self.status])
if self.headers:
s += self.headers.to_string() + "\r\n"
s += "\r\n"
s += self.body
return s

7
scrapy/tests/test_http_request.py
View File

@ -166,13 +166,6 @@ class RequestTest(unittest.TestCase):
self.assertEqual(r4.meta, {})
assert r4.dont_filter is False
def test_httprepr(self):
r1 = Request("http://www.example.com")
self.assertEqual(r1.httprepr(), 'GET http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
r1 = Request("http://www.example.com", method='POST', headers={"Content-type": "text/html"}, body="Some body")
self.assertEqual(r1.httprepr(), 'POST http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
class FormRequestTest(unittest.TestCase):

7
scrapy/tests/test_http_response.py
View File

@ -104,13 +104,6 @@ class ResponseTest(unittest.TestCase):
self.assertEqual(r4.meta, {})
self.assertEqual(r4.flags, [])
def test_httprepr(self):
r1 = Response("http://www.example.com")
self.assertEqual(r1.httprepr(), 'HTTP/1.1 200 OK\r\n\r\n')
r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body="Some body")
self.assertEqual(r1.httprepr(), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
def test_encoding(self):
unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'
self.assertRaises(TypeError, Response, 'http://www.example.com', body=u'unicode body')

14
scrapy/tests/test_utils_request.py
View File

@ -1,14 +1,10 @@
import unittest
from scrapy.http import Request
from scrapy.utils.request import request_fingerprint, request_authenticate
from scrapy.utils.request import request_fingerprint, request_authenticate, request_httprepr
class UtilsRequestTest(unittest.TestCase):
def test_request_fingerprint(self):
url = 'http://www.scrapy.org'
r = Request(url=url)
urlhash = request_fingerprint(r)
r1 = Request("http://www.example.com/query?id=111&cat=222")
r2 = Request("http://www.example.com/query?cat=222&id=111")
self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
@ -59,11 +55,17 @@ class UtilsRequestTest(unittest.TestCase):
fp2 = request_fingerprint(r2)
self.assertNotEqual(fp1, fp2)
def test_request_authenticate(self):
r = Request("http://www.example.com")
request_authenticate(r, 'someuser', 'somepass')
self.assertEqual(r.headers['Authorization'], 'Basic c29tZXVzZXI6c29tZXBhc3M=')
def test_request_httprepr(self):
r1 = Request("http://www.example.com")
self.assertEqual(request_httprepr(r1), 'GET http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
r1 = Request("http://www.example.com", method='POST', headers={"Content-type": "text/html"}, body="Some body")
self.assertEqual(request_httprepr(r1), 'POST http://www.example.com HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
if __name__ == "__main__":
unittest.main()

16
scrapy/tests/test_utils_response.py
View File

@ -1,6 +1,6 @@
import unittest
from scrapy.http import Response, TextResponse
from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh
from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh, response_httprepr
class ResponseUtilsTest(unittest.TestCase):
dummy_response = TextResponse(url='http://example.org/', body='dummy_response')
@ -33,7 +33,7 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertEqual(get_base_url(response), 'http://example.org/something')
def test_get_meta_refresh(self):
body="""
body = """
<html>
<head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
<body>blahablsdfsal&amp;</body>
@ -42,14 +42,22 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
# refresh without url should return (None, None)
body="""<meta http-equiv="refresh" content="5" />"""
body = """<meta http-equiv="refresh" content="5" />"""
response = Response(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), (None, None))
body="""<meta http-equiv="refresh" content="5;
body = """<meta http-equiv="refresh" content="5;
url=http://example.org/newpage" /></head>"""
response = Response(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
def test_response_httprepr(self):
r1 = Response("http://www.example.com")
self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n')
r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body="Some body")
self.assertEqual(response_httprepr(r1), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
if __name__ == "__main__":
unittest.main()

14
scrapy/utils/request.py
View File

@ -72,3 +72,17 @@ def request_info(request):
fp = request_fingerprint(request)
return "<Request: %s %s (%s..)>" % (request.method, request.url, fp[:8])
def request_httprepr(request):
"""Return the raw HTTP representation (as string) of the given request.
This is provided only for reference since it's not the actual stream of
bytes that will be send when performing the request (that's controlled
by Twisted).
"""
s = "%s %s HTTP/1.1\r\n" % (request.method, request.url)
s += "Host: %s\r\n" % request.url.hostname
if request.headers:
s += request.headers.to_string() + "\r\n"
s += "\r\n"
s += request.body
return s

16
scrapy/utils/response.py
View File

@ -4,7 +4,10 @@ scrapy.http.Response objects
"""
import re
from twisted.web import http
from twisted.web.http import RESPONSES
from scrapy.http.response import Response
def body_or_str(obj, unicode=True):
@ -48,3 +51,16 @@ def response_status_message(status):
404 Not Found
"""
return '%s %s' % (status, http.responses.get(int(status)))
def response_httprepr(response):
"""Return raw HTTP representation (as string) of the given response. This
is provided only for reference, since it's not the exact stream of bytes
that was received (that's not exposed by Twisted).
"""
s = "HTTP/1.1 %d %s\r\n" % (response.status, RESPONSES[response.status])
if response.headers:
s += response.headers.to_string() + "\r\n"
s += "\r\n"
s += response.body
return s