From 47d3c63338c4550341cc7518bcb48d0fc606b67d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:28:22 +0300 Subject: [PATCH] py3: port fetch and shell commands, and review + enable already passing test_closespider.py and tests/test_utils_template.py --- scrapy/commands/fetch.py | 6 ++++-- scrapy/utils/testsite.py | 12 ++++++------ tests/py3-ignores.txt | 4 ---- tests/test_command_fetch.py | 8 ++++---- tests/test_command_shell.py | 14 +++++++------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index e61eedf50..3888da210 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError from scrapy.utils.spider import spidercls_for_request, DefaultSpider +from scrapy.utils.python import to_unicode class Command(ScrapyCommand): @@ -30,7 +31,8 @@ class Command(ScrapyCommand): def _print_headers(self, headers, prefix): for key, values in headers.items(): for value in values: - print('%s %s: %s' % (prefix, key, value)) + print('%s %s: %s' % ( + prefix, to_unicode(key), to_unicode(value))) def _print_response(self, response, opts): if opts.headers: @@ -38,7 +40,7 @@ class Command(ScrapyCommand): print('>') self._print_headers(response.headers, '<') else: - print(response.body) + print(to_unicode(response.body)) def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 01508bdb4..ad0375443 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -22,13 +22,13 @@ class SiteTest(object): def test_site(): r = resource.Resource() - r.putChild("text", static.Data("Works", "text/plain")) - r.putChild("html", static.Data("

Works

World

", "text/html")) - r.putChild("enc-gb18030", static.Data("

gb18030 encoding

", "text/html; charset=gb18030")) - r.putChild("redirect", util.Redirect("/redirected")) - r.putChild("redirected", static.Data("Redirected here", "text/plain")) + r.putChild(b"text", static.Data(b"Works", "text/plain")) + r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) + r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) - + if __name__ == '__main__': port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 1f7d85ef7..1f0f34c49 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,6 +1,3 @@ -tests/test_closespider.py -tests/test_command_fetch.py -tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py @@ -12,7 +9,6 @@ tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py -tests/test_utils_template.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 5283852b7..4843a9a2f 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url('/text')]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace('\r', '') # required on win32 - assert 'Server: TwistedWeb' in out - assert 'Content-Type: text/plain' in out + out = out.replace(b'\r', b'') # required on win32 + assert b'Server: TwistedWeb' in out, out + assert b'Content-Type: text/plain' in out diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index a56236d54..105202754 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(['-c', 'item']) - assert '{}' in out + assert b'{}' in out @defer.inlineCallbacks def test_response_body(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body']) - assert 'Works' in out + assert b'Works' in out @defer.inlineCallbacks def test_response_type_text(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)']) - assert 'TextResponse' in out + assert b'TextResponse' in out @defer.inlineCallbacks def test_response_type_html(self): _, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)']) - assert 'HtmlResponse' in out + assert b'HtmlResponse' in out @defer.inlineCallbacks def test_response_selector_html(self): xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding']) - self.assertEqual(out.strip(), 'gb18030') + self.assertEqual(out.strip(), b'gb18030') @defer.inlineCallbacks def test_redirect(self): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) - assert out.strip().endswith('/redirected') + assert out.strip().endswith(b'/redirected') @defer.inlineCallbacks def test_request_replace(self):