diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index e61eedf50..f09a873c1 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,4 +1,5 @@ from __future__ import print_function +import sys, six from w3lib.url import is_url from scrapy.commands import ScrapyCommand @@ -30,15 +31,19 @@ class Command(ScrapyCommand): def _print_headers(self, headers, prefix): for key, values in headers.items(): for value in values: - print('%s %s: %s' % (prefix, key, value)) + self._print_bytes(prefix + b' ' + key + b': ' + value) def _print_response(self, response, opts): if opts.headers: - self._print_headers(response.request.headers, '>') + self._print_headers(response.request.headers, b'>') print('>') - self._print_headers(response.headers, '<') + self._print_headers(response.headers, b'<') else: - print(response.body) + self._print_bytes(response.body) + + def _print_bytes(self, bytes_): + bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer + bytes_writer.write(bytes_ + b'\n') def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 01508bdb4..ad0375443 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -22,13 +22,13 @@ class SiteTest(object): def test_site(): r = resource.Resource() - r.putChild("text", static.Data("Works", "text/plain")) - r.putChild("html", static.Data("

Works

World

", "text/html")) - r.putChild("enc-gb18030", static.Data("

gb18030 encoding

", "text/html; charset=gb18030")) - r.putChild("redirect", util.Redirect("/redirected")) - r.putChild("redirected", static.Data("Redirected here", "text/plain")) + r.putChild(b"text", static.Data(b"Works", "text/plain")) + r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) + r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) - + if __name__ == '__main__': port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 3e147e9e9..8c883ee3c 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,6 +1,4 @@ tests/test_closespider.py -tests/test_command_fetch.py -tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py @@ -11,7 +9,6 @@ tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py -tests/test_utils_template.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 5283852b7..4843a9a2f 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url('/text')]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace('\r', '') # required on win32 - assert 'Server: TwistedWeb' in out - assert 'Content-Type: text/plain' in out + out = out.replace(b'\r', b'') # required on win32 + assert b'Server: TwistedWeb' in out, out + assert b'Content-Type: text/plain' in out diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index a56236d54..105202754 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(['-c', 'item']) - assert '{}' in out + assert b'{}' in out @defer.inlineCallbacks def test_response_body(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body']) - assert 'Works' in out + assert b'Works' in out @defer.inlineCallbacks def test_response_type_text(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)']) - assert 'TextResponse' in out + assert b'TextResponse' in out @defer.inlineCallbacks def test_response_type_html(self): _, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)']) - assert 'HtmlResponse' in out + assert b'HtmlResponse' in out @defer.inlineCallbacks def test_response_selector_html(self): xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding']) - self.assertEqual(out.strip(), 'gb18030') + self.assertEqual(out.strip(), b'gb18030') @defer.inlineCallbacks def test_redirect(self): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) - assert out.strip().endswith('/redirected') + assert out.strip().endswith(b'/redirected') @defer.inlineCallbacks def test_request_replace(self):