From 7f927f68e1d3eae71550a48c5484099ca7dd9ce1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 1 Aug 2015 00:35:43 +0500 Subject: [PATCH] PY3 port scrapy.utils.response --- scrapy/utils/response.py | 29 ++++++++++++--------- tests/py3-ignores.txt | 1 - tests/test_utils_response.py | 49 +++++++++++++++++++++++------------- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b5d7a58ca..3d1af7e51 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -2,7 +2,6 @@ This module provides some useful functions for working with scrapy.http.Response objects """ - import os import re import weakref @@ -11,6 +10,7 @@ import tempfile from twisted.web import http from twisted.web.http import RESPONSES +from scrapy.utils.python import to_bytes from w3lib import html from scrapy.utils.decorators import deprecated @@ -27,10 +27,11 @@ def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.body_as_unicode()[0:4096] - _baseurl_cache[response] = html.get_base_url(text, response.url, \ + _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) return _baseurl_cache[response] + _noscript_re = re.compile(u'', re.IGNORECASE | re.DOTALL) _script_re = re.compile(u'.*?', re.IGNORECASE | re.DOTALL) _metaref_cache = weakref.WeakKeyDictionary() @@ -40,10 +41,11 @@ def get_meta_refresh(response): text = response.body_as_unicode()[0:4096] text = _noscript_re.sub(u'', text) text = _script_re.sub(u'', text) - _metaref_cache[response] = html.get_meta_refresh(text, response.url, \ + _metaref_cache[response] = html.get_meta_refresh(text, response.url, response.encoding) return _metaref_cache[response] + def response_status_message(status): """Return status code plus status text descriptive message @@ -55,19 +57,21 @@ def response_status_message(status): """ return '%s %s' % (status, http.responses.get(int(status))) + def response_httprepr(response): - """Return raw HTTP representation (as string) of the given response. This + """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). """ - - s = "HTTP/1.1 %d %s\r\n" % (response.status, RESPONSES.get(response.status, '')) + s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \ + to_bytes(RESPONSES.get(response.status, b'')) + b"\r\n" if response.headers: - s += response.headers.to_string() + "\r\n" - s += "\r\n" + s += response.headers.to_string() + b"\r\n" + s += b"\r\n" s += response.body return s + def open_in_browser(response, _openfunc=webbrowser.open): """Open the given response in a local web browser, populating the tag for external links to work @@ -76,14 +80,15 @@ def open_in_browser(response, _openfunc=webbrowser.open): # XXX: this implementation is a bit dirty and could be improved body = response.body if isinstance(response, HtmlResponse): - if '', '' % response.url) + if b'', to_bytes(repl)) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' else: - raise TypeError("Unsupported response type: %s" % \ - response.__class__.__name__) + raise TypeError("Unsupported response type: %s" % + response.__class__.__name__) fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index c51bc5981..432b58cac 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -42,7 +42,6 @@ tests/test_stats.py tests/test_utils_iterators.py tests/test_utils_log.py tests/test_utils_reqser.py -tests/test_utils_response.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 92c92c057..1d1638e06 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -3,53 +3,59 @@ import unittest from six.moves.urllib.parse import urlparse from scrapy.http import Response, TextResponse, HtmlResponse -from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh +from scrapy.utils.python import to_bytes +from scrapy.utils.response import (response_httprepr, open_in_browser, + get_meta_refresh, get_base_url) __doctests__ = ['scrapy.utils.response'] + class ResponseUtilsTest(unittest.TestCase): - dummy_response = TextResponse(url='http://example.org/', body='dummy_response') + dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response') def test_response_httprepr(self): r1 = Response("http://www.example.com") - self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n') + self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n') - r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body="Some body") - self.assertEqual(response_httprepr(r1), 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body") + self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') - r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body="Some body") - self.assertEqual(response_httprepr(r1), 'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body") + self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body') def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" - body = " test page test body " + body = b" test page test body " + def browser_open(burl): path = urlparse(burl).path if not os.path.exists(path): path = burl.replace('file://', '') - bbody = open(path).read() - assert '' % url in bbody, " tag not added" + with open(path, "rb") as f: + bbody = f.read() + self.assertIn(b'', bbody) return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=browser_open), \ "Browser not called" - self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \ - debug=True) + + resp = Response(url, body=body) + self.assertRaises(TypeError, open_in_browser, resp, debug=True) def test_get_meta_refresh(self): - r1 = HtmlResponse("http://www.example.com", body=""" + r1 = HtmlResponse("http://www.example.com", body=b""" Dummy blahablsdfsal& """) - r2 = HtmlResponse("http://www.example.com", body=""" + r2 = HtmlResponse("http://www.example.com", body=b""" Dummy blahablsdfsal& """) - r3 = HtmlResponse("http://www.example.com", body=""" + r3 = HtmlResponse("http://www.example.com", body=b"""