RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1693 from lopuhin/py3-commands-fetch-shell 

[MRG+1] Py3: port commands fetch and shell
This commit is contained in:
Paul Tremberth 2016-01-21 10:59:32 +01:00
parent 6d8620ffe7 fd24e22442
commit da83f2dca7
5 changed files with 26 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scrapy/commands/fetch.py
View File

@ -1,4 +1,5 @@
from __future__ import print_function
import sys, six
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
@ -30,15 +31,19 @@ class Command(ScrapyCommand):
def _print_headers(self, headers, prefix):
for key, values in headers.items():
for value in values:
print('%s %s: %s' % (prefix, key, value))
self._print_bytes(prefix + b' ' + key + b': ' + value)
def _print_response(self, response, opts):
if opts.headers:
self._print_headers(response.request.headers, '>')
self._print_headers(response.request.headers, b'>')
print('>')
self._print_headers(response.headers, '<')
self._print_headers(response.headers, b'<')
else:
print(response.body)
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
bytes_writer.write(bytes_ + b'\n')
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):

12
scrapy/utils/testsite.py
View File

@ -22,13 +22,13 @@ class SiteTest(object):
def test_site():
r = resource.Resource()
r.putChild("text", static.Data("Works", "text/plain"))
r.putChild("html", static.Data("<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
r.putChild("enc-gb18030", static.Data("<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
r.putChild("redirect", util.Redirect("/redirected"))
r.putChild("redirected", static.Data("Redirected here", "text/plain"))
r.putChild(b"text", static.Data(b"Works", "text/plain"))
r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
r.putChild(b"redirect", util.Redirect(b"/redirected"))
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
return server.Site(r)
if __name__ == '__main__':
port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")

3
tests/py3-ignores.txt
View File

@ -1,6 +1,4 @@
tests/test_closespider.py
tests/test_command_fetch.py
tests/test_command_shell.py
tests/test_exporters.py
tests/test_linkextractors_deprecated.py
tests/test_crawl.py
@ -11,7 +9,6 @@ tests/test_pipeline_files.py
tests/test_pipeline_images.py
tests/test_proxy_connect.py
tests/test_spidermiddleware_httperror.py
tests/test_utils_template.py
scrapy/xlib/tx/iweb.py
scrapy/xlib/tx/interfaces.py

8
tests/test_command_fetch.py
View File

@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_output(self):
_, out, _ = yield self.execute([self.url('/text')])
self.assertEqual(out.strip(), 'Works')
self.assertEqual(out.strip(), b'Works')
@defer.inlineCallbacks
def test_headers(self):
_, out, _ = yield self.execute([self.url('/text'), '--headers'])
out = out.replace('\r', '') # required on win32
assert 'Server: TwistedWeb' in out
assert 'Content-Type: text/plain' in out
out = out.replace(b'\r', b'') # required on win32
assert b'Server: TwistedWeb' in out, out
assert b'Content-Type: text/plain' in out

14
tests/test_command_shell.py
View File

@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_empty(self):
_, out, _ = yield self.execute(['-c', 'item'])
assert '{}' in out
assert b'{}' in out
@defer.inlineCallbacks
def test_response_body(self):
_, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])
assert 'Works' in out
assert b'Works' in out
@defer.inlineCallbacks
def test_response_type_text(self):
_, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])
assert 'TextResponse' in out
assert b'TextResponse' in out
@defer.inlineCallbacks
def test_response_type_html(self):
_, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])
assert 'HtmlResponse' in out
assert b'HtmlResponse' in out
@defer.inlineCallbacks
def test_response_selector_html(self):
xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
self.assertEqual(out.strip(), 'Works')
self.assertEqual(out.strip(), b'Works')
@defer.inlineCallbacks
def test_response_encoding_gb18030(self):
_, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])
self.assertEqual(out.strip(), 'gb18030')
self.assertEqual(out.strip(), b'gb18030')
@defer.inlineCallbacks
def test_redirect(self):
_, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
assert out.strip().endswith('/redirected')
assert out.strip().endswith(b'/redirected')
@defer.inlineCallbacks
def test_request_replace(self):