mirror of https://github.com/scrapy/scrapy.git
139 lines
5.3 KiB
Python
139 lines
5.3 KiB
Python
from os.path import join
|
|
|
|
from twisted.trial import unittest
|
|
from twisted.internet import defer
|
|
|
|
from scrapy.commands.shell import guess_scheme
|
|
from scrapy.utils.testsite import SiteTest
|
|
from scrapy.utils.testproc import ProcessTest
|
|
|
|
from tests import tests_datadir
|
|
|
|
|
|
class ShellURLTest(unittest.TestCase):
|
|
pass
|
|
|
|
def create_guess_scheme_t(args):
|
|
def do_expected(self):
|
|
url = guess_scheme(args[0])
|
|
assert url.startswith(args[1]), \
|
|
'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
|
|
args[0], url, args[1])
|
|
return do_expected
|
|
|
|
def create_skipped_scheme_t(args):
|
|
def do_expected(self):
|
|
raise unittest.SkipTest(args[2])
|
|
url = guess_scheme(args[0])
|
|
assert url.startswith(args[1])
|
|
return do_expected
|
|
|
|
for k, args in enumerate ([
|
|
('/index', 'file://'),
|
|
('/index.html', 'file://'),
|
|
('./index.html', 'file://'),
|
|
('../index.html', 'file://'),
|
|
('../../index.html', 'file://'),
|
|
('./data/index.html', 'file://'),
|
|
('.hidden/data/index.html', 'file://'),
|
|
('/home/user/www/index.html', 'file://'),
|
|
('//home/user/www/index.html', 'file://'),
|
|
('file:///home/user/www/index.html', 'file://'),
|
|
|
|
('index.html', 'http://'),
|
|
('example.com', 'http://'),
|
|
('www.example.com', 'http://'),
|
|
('www.example.com/index.html', 'http://'),
|
|
('http://example.com', 'http://'),
|
|
('http://example.com/index.html', 'http://'),
|
|
('localhost', 'http://'),
|
|
('localhost/index.html', 'http://'),
|
|
|
|
# some corner cases (default to http://)
|
|
('/', 'http://'),
|
|
('.../test', 'http://'),
|
|
|
|
], start=1):
|
|
t_method = create_guess_scheme_t(args)
|
|
t_method.__name__ = 'test_uri_%03d' % k
|
|
setattr (ShellURLTest, t_method.__name__, t_method)
|
|
|
|
# TODO: the following tests do not pass with current implementation
|
|
for k, args in enumerate ([
|
|
('C:\absolute\path\to\a\file.html', 'file://',
|
|
'Windows filepath are not supported for scrapy shell'),
|
|
], start=1):
|
|
t_method = create_skipped_scheme_t(args)
|
|
t_method.__name__ = 'test_uri_skipped_%03d' % k
|
|
setattr (ShellURLTest, t_method.__name__, t_method)
|
|
|
|
|
|
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
|
|
|
|
command = 'shell'
|
|
|
|
@defer.inlineCallbacks
|
|
def test_empty(self):
|
|
_, out, _ = yield self.execute(['-c', 'item'])
|
|
assert b'{}' in out
|
|
|
|
@defer.inlineCallbacks
|
|
def test_response_body(self):
|
|
_, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])
|
|
assert b'Works' in out
|
|
|
|
@defer.inlineCallbacks
|
|
def test_response_type_text(self):
|
|
_, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])
|
|
assert b'TextResponse' in out
|
|
|
|
@defer.inlineCallbacks
|
|
def test_response_type_html(self):
|
|
_, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])
|
|
assert b'HtmlResponse' in out
|
|
|
|
@defer.inlineCallbacks
|
|
def test_response_selector_html(self):
|
|
xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
|
|
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
|
|
self.assertEqual(out.strip(), b'Works')
|
|
|
|
@defer.inlineCallbacks
|
|
def test_response_encoding_gb18030(self):
|
|
_, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])
|
|
self.assertEqual(out.strip(), b'gb18030')
|
|
|
|
@defer.inlineCallbacks
|
|
def test_redirect(self):
|
|
_, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
|
|
assert out.strip().endswith(b'/redirected')
|
|
|
|
@defer.inlineCallbacks
|
|
def test_request_replace(self):
|
|
url = self.url('/text')
|
|
code = "fetch('{0}') or fetch(response.request.replace(method='POST'))"
|
|
errcode, out, _ = yield self.execute(['-c', code.format(url)])
|
|
self.assertEqual(errcode, 0, out)
|
|
|
|
@defer.inlineCallbacks
|
|
def test_local_file(self):
|
|
filepath = join(tests_datadir, 'test_site/index.html')
|
|
_, out, _ = yield self.execute([filepath, '-c', 'item'])
|
|
assert b'{}' in out
|
|
|
|
@defer.inlineCallbacks
|
|
def test_local_nofile(self):
|
|
filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
|
|
errcode, out, err = yield self.execute([filepath, '-c', 'item'],
|
|
check_code=False)
|
|
self.assertEqual(errcode, 1, out or err)
|
|
self.assertIn(b'No such file or directory', err)
|
|
|
|
@defer.inlineCallbacks
|
|
def test_dns_failures(self):
|
|
url = 'www.somedomainthatdoesntexi.st'
|
|
errcode, out, err = yield self.execute([url, '-c', 'item'],
|
|
check_code=False)
|
|
self.assertEqual(errcode, 1, out or err)
|
|
self.assertIn(b'DNS lookup failed', err)
|