mirror of https://github.com/scrapy/scrapy.git
py3: port fetch and shell commands, and review + enable already passing test_closespider.py and tests/test_utils_template.py
This commit is contained in:
Konstantin Lopuhin
parent
56b69d2ea8
commit
47d3c63338
|
|
@ -5,6 +5,7 @@ from scrapy.commands import ScrapyCommand
|
|||
from scrapy.http import Request
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
|
|
@ -30,7 +31,8 @@ class Command(ScrapyCommand):
|
|||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
for value in values:
|
||||
print('%s %s: %s' % (prefix, key, value))
|
||||
print('%s %s: %s' % (
|
||||
prefix, to_unicode(key), to_unicode(value)))
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
if opts.headers:
|
||||
|
|
@ -38,7 +40,7 @@ class Command(ScrapyCommand):
|
|||
print('>')
|
||||
self._print_headers(response.headers, '<')
|
||||
else:
|
||||
print(response.body)
|
||||
print(to_unicode(response.body))
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1 or not is_url(args[0]):
|
||||
|
|
|
|||
|
|
@ -22,13 +22,13 @@ class SiteTest(object):
|
|||
|
||||
def test_site():
|
||||
r = resource.Resource()
|
||||
r.putChild("text", static.Data("Works", "text/plain"))
|
||||
r.putChild("html", static.Data("<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
|
||||
r.putChild("enc-gb18030", static.Data("<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
|
||||
r.putChild("redirect", util.Redirect("/redirected"))
|
||||
r.putChild("redirected", static.Data("Redirected here", "text/plain"))
|
||||
r.putChild(b"text", static.Data(b"Works", "text/plain"))
|
||||
r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
|
||||
r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
|
||||
r.putChild(b"redirect", util.Redirect(b"/redirected"))
|
||||
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
|
||||
return server.Site(r)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
tests/test_closespider.py
|
||||
tests/test_command_fetch.py
|
||||
tests/test_command_shell.py
|
||||
tests/test_exporters.py
|
||||
tests/test_linkextractors_deprecated.py
|
||||
tests/test_crawl.py
|
||||
|
|
@ -12,7 +9,6 @@ tests/test_pipeline_files.py
|
|||
tests/test_pipeline_images.py
|
||||
tests/test_proxy_connect.py
|
||||
tests/test_spidermiddleware_httperror.py
|
||||
tests/test_utils_template.py
|
||||
|
||||
scrapy/xlib/tx/iweb.py
|
||||
scrapy/xlib/tx/interfaces.py
|
||||
|
|
|
|||
|
|
@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_output(self):
|
||||
_, out, _ = yield self.execute([self.url('/text')])
|
||||
self.assertEqual(out.strip(), 'Works')
|
||||
self.assertEqual(out.strip(), b'Works')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_headers(self):
|
||||
_, out, _ = yield self.execute([self.url('/text'), '--headers'])
|
||||
out = out.replace('\r', '') # required on win32
|
||||
assert 'Server: TwistedWeb' in out
|
||||
assert 'Content-Type: text/plain' in out
|
||||
out = out.replace(b'\r', b'') # required on win32
|
||||
assert b'Server: TwistedWeb' in out, out
|
||||
assert b'Content-Type: text/plain' in out
|
||||
|
|
|
|||
|
|
@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_empty(self):
|
||||
_, out, _ = yield self.execute(['-c', 'item'])
|
||||
assert '{}' in out
|
||||
assert b'{}' in out
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_body(self):
|
||||
_, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])
|
||||
assert 'Works' in out
|
||||
assert b'Works' in out
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_type_text(self):
|
||||
_, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])
|
||||
assert 'TextResponse' in out
|
||||
assert b'TextResponse' in out
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_type_html(self):
|
||||
_, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])
|
||||
assert 'HtmlResponse' in out
|
||||
assert b'HtmlResponse' in out
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_selector_html(self):
|
||||
xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
|
||||
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
|
||||
self.assertEqual(out.strip(), 'Works')
|
||||
self.assertEqual(out.strip(), b'Works')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_encoding_gb18030(self):
|
||||
_, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])
|
||||
self.assertEqual(out.strip(), 'gb18030')
|
||||
self.assertEqual(out.strip(), b'gb18030')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_redirect(self):
|
||||
_, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
|
||||
assert out.strip().endswith('/redirected')
|
||||
assert out.strip().endswith(b'/redirected')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_request_replace(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue