mirror of https://github.com/scrapy/scrapy.git
E501 compliance (tests)
This commit is contained in:
Eugenio Lacuesta
parent
36a3913a41
commit
182394bcec
129
pytest.ini
129
pytest.ini
|
|
@ -108,84 +108,63 @@ flake8-ignore =
|
|||
scrapy/spiderloader.py F841 E126
|
||||
scrapy/squeues.py E128
|
||||
# tests
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/__init__.py E402
|
||||
tests/mockserver.py E401 E126 E123
|
||||
tests/pipelines.py F841
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py E128 E501
|
||||
tests/test_contracts.py E501 E128
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E501
|
||||
tests/test_dependencies.py F841 E501
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
|
||||
tests/spiders.py E127
|
||||
tests/test_closespider.py E127
|
||||
tests/test_command_parse.py E128
|
||||
tests/test_command_shell.py E128
|
||||
tests/test_commands.py E128
|
||||
tests/test_contracts.py E128
|
||||
tests/test_crawl.py E741 E265
|
||||
tests/test_crawler.py F841
|
||||
tests/test_dependencies.py F841
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E265 E126 E123
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E128 E265 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128 E127
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E501 E741 E128 E124
|
||||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_exporters.py E501 E731 E128 E124
|
||||
tests/test_downloadermiddleware_httpcompression.py E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E128
|
||||
tests/test_downloadermiddleware_redirect.py E128 E127
|
||||
tests/test_downloadermiddleware_retry.py E128 E126
|
||||
tests/test_dupefilters.py E741 E128 E124
|
||||
tests/test_engine.py E401 E128
|
||||
tests/test_exporters.py E731 E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
|
||||
tests/test_http_response.py E501 E128 E265
|
||||
tests/test_feedexport.py F841 E241
|
||||
tests/test_http_request.py E402 E127 E128 E128 E126 E123
|
||||
tests/test_http_response.py E128 E265
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E501 E731 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_mail.py E128 E501
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E501 E128 E126
|
||||
tests/test_pipeline_files.py E501
|
||||
tests/test_pipeline_images.py F841 E501
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E502
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501
|
||||
tests/test_robotstxt_interface.py E501 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111
|
||||
tests/test_spidermiddleware_output_chain.py E501
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E741
|
||||
tests/test_utils_asyncio.py E501
|
||||
tests/test_utils_conf.py E501 E128
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501
|
||||
tests/test_utils_defer.py E501 F841
|
||||
tests/test_utils_deprecate.py F841 E501
|
||||
tests/test_utils_http.py E501 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129 E241
|
||||
tests/test_linkextractors.py E128 E124
|
||||
tests/test_loader.py E731 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E122
|
||||
tests/test_mail.py E128
|
||||
tests/test_middleware.py E128
|
||||
tests/test_pipeline_crawl.py E128 E126
|
||||
tests/test_pipeline_images.py F841
|
||||
tests/test_pipeline_media.py E741 E731 E128 E502
|
||||
tests/test_proxy_connect.py E741
|
||||
tests/test_scheduler.py E126 E123
|
||||
tests/test_selector.py E127
|
||||
tests/test_spidermiddleware_httperror.py E128 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E128 E111
|
||||
tests/test_spidermiddleware_referer.py F841 E125 E201 E124 E241 E121
|
||||
tests/test_squeues.py E741
|
||||
tests/test_utils_conf.py E128
|
||||
tests/test_utils_datatypes.py E402
|
||||
tests/test_utils_defer.py F841
|
||||
tests/test_utils_deprecate.py F841
|
||||
tests/test_utils_http.py E128 W504
|
||||
tests/test_utils_iterators.py E128 E129 E241
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E501 E731
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_python.py E731
|
||||
tests/test_utils_reqser.py E128
|
||||
tests/test_utils_request.py E128
|
||||
tests/test_utils_signal.py E741 F841 E731
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
tests/test_utils_misc/__init__.py E501
|
||||
tests/test_utils_sitemap.py E128 E124
|
||||
tests/test_utils_url.py E127 E125 E241 E126 E123
|
||||
tests/test_webclient.py E128 E122 E402 E241 E123 E126
|
||||
tests/test_cmdline/__init__.py
|
||||
tests/test_settings/__init__.py E128
|
||||
tests/test_spiderloader/__init__.py E128
|
||||
tests/test_utils_misc/__init__.py
|
||||
|
|
|
|||
|
|
@ -56,7 +56,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_redirect_not_follow_302(self):
|
||||
_, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
|
||||
_, out, _ = yield self.execute(
|
||||
['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']
|
||||
)
|
||||
assert out.strip().endswith(b'302')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
|
|||
|
|
@ -303,8 +303,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
|
|||
def test_ipv6_default_name_resolver(self):
|
||||
log = self.run_script('default_name_resolver.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log)
|
||||
self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
|
||||
self.assertIn(
|
||||
"twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
|
||||
log)
|
||||
|
||||
def test_ipv6_alternative_name_resolver(self):
|
||||
log = self.run_script('alternative_name_resolver.py')
|
||||
|
|
|
|||
|
|
@ -493,7 +493,10 @@ class Http11TestCase(HttpTestCase):
|
|||
class Https11TestCase(Http11TestCase):
|
||||
scheme = 'https'
|
||||
|
||||
tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"'
|
||||
tls_log_message = (
|
||||
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
|
||||
'subject "/C=IE/O=Scrapy/CN=localhost"'
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_tls_logging(self):
|
||||
|
|
@ -542,7 +545,10 @@ class Https11InvalidDNSPattern(Https11TestCase):
|
|||
from service_identity.exceptions import CertificateError # noqa: F401
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("cryptography lib is too old")
|
||||
self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
|
||||
self.tls_log_message = (
|
||||
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
|
||||
'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
|
||||
)
|
||||
super(Https11InvalidDNSPattern, self).setUp()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -124,7 +124,8 @@ class HttpCompressionTest(TestCase):
|
|||
'Content-Encoding': 'gzip',
|
||||
}
|
||||
f = BytesIO()
|
||||
plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
|
||||
plainbody = (b'<html><head><title>Some page</title>'
|
||||
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
|
||||
zf = GzipFile(fileobj=f, mode='wb')
|
||||
zf.write(plainbody)
|
||||
zf.close()
|
||||
|
|
@ -142,7 +143,8 @@ class HttpCompressionTest(TestCase):
|
|||
'Content-Encoding': 'gzip',
|
||||
}
|
||||
f = BytesIO()
|
||||
plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
|
||||
plainbody = (b'<html><head><title>Some page</title>'
|
||||
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
|
||||
zf = GzipFile(fileobj=f, mode='wb')
|
||||
zf.write(plainbody)
|
||||
zf.close()
|
||||
|
|
@ -158,7 +160,8 @@ class HttpCompressionTest(TestCase):
|
|||
headers = {
|
||||
'Content-Encoding': 'identity',
|
||||
}
|
||||
plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
|
||||
plainbody = (b'<html><head><title>Some page</title>'
|
||||
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">')
|
||||
respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody)
|
||||
response = respcls("http://www.example.com/index", headers=headers, body=plainbody)
|
||||
request = Request("http://www.example.com/index")
|
||||
|
|
|
|||
|
|
@ -151,7 +151,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
|
||||
self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
|
||||
self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
|
||||
self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
|
||||
self.assertEqual(
|
||||
req3.meta['redirect_urls'],
|
||||
['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
|
||||
)
|
||||
|
||||
def test_redirect_reasons(self):
|
||||
req1 = Request('http://scrapytest.org/first')
|
||||
|
|
@ -282,7 +285,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
|
|||
self.assertEqual(req2.url, 'http://scrapytest.org/redirected')
|
||||
self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first'])
|
||||
self.assertEqual(req3.url, 'http://scrapytest.org/redirected2')
|
||||
self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'])
|
||||
self.assertEqual(
|
||||
req3.meta['redirect_urls'],
|
||||
['http://scrapytest.org/first', 'http://scrapytest.org/redirected']
|
||||
)
|
||||
|
||||
def test_redirect_reasons(self):
|
||||
req1 = Request('http://scrapytest.org/first')
|
||||
|
|
|
|||
|
|
@ -111,7 +111,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(type(exported), dict)
|
||||
self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'})
|
||||
self.assertEqual(
|
||||
exported,
|
||||
{'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
|
||||
)
|
||||
self.assertEqual(type(exported['age']), dict)
|
||||
self.assertEqual(type(exported['age']['age']), dict)
|
||||
|
||||
|
|
@ -121,7 +124,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
i3 = TestItem(name=u'Jesus', age=[i2])
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
|
||||
self.assertEqual(
|
||||
exported,
|
||||
{'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
|
||||
)
|
||||
self.assertEqual(type(exported['age'][0]), dict)
|
||||
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
|
||||
|
||||
|
|
@ -131,7 +137,10 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
i3 = TestItem(name=u'Jesus', age=[i2])
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
|
||||
self.assertEqual(
|
||||
exported,
|
||||
{'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
|
||||
)
|
||||
self.assertEqual(type(exported['age'][0]), dict)
|
||||
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
|
||||
|
||||
|
|
@ -327,13 +336,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
self.assertXmlEquivalent(fp.getvalue(), expected_value)
|
||||
|
||||
def _check_output(self):
|
||||
expected_value = b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
|
||||
expected_value = (
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
|
||||
)
|
||||
self.assertXmlEquivalent(self.output.getvalue(), expected_value)
|
||||
|
||||
def test_multivalued_fields(self):
|
||||
self.assertExportResult(
|
||||
TestItem(name=[u'John\xa3', u'Doe']),
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
|
||||
(
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
|
||||
)
|
||||
)
|
||||
|
||||
def test_nested_item(self):
|
||||
|
|
|
|||
|
|
@ -718,10 +718,13 @@ class FeedExportTest(unittest.TestCase):
|
|||
header = ['foo']
|
||||
|
||||
formats = {
|
||||
'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
|
||||
'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
|
||||
'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
|
||||
'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
|
||||
'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
|
||||
'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
|
||||
'xml': (
|
||||
'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
'<items><item><foo>Test\xd6</foo></item></items>'
|
||||
).encode('utf-8'),
|
||||
'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'),
|
||||
}
|
||||
|
||||
for fmt, expected in formats.items():
|
||||
|
|
@ -735,10 +738,13 @@ class FeedExportTest(unittest.TestCase):
|
|||
self.assertEqual(expected, data[fmt])
|
||||
|
||||
formats = {
|
||||
'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
|
||||
'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
|
||||
'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
|
||||
'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
|
||||
'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'),
|
||||
'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'),
|
||||
'xml': (
|
||||
'<?xml version="1.0" encoding="latin-1"?>\n'
|
||||
'<items><item><foo>Test\xd6</foo></item></items>'
|
||||
).encode('latin-1'),
|
||||
'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'),
|
||||
}
|
||||
|
||||
for fmt, expected in formats.items():
|
||||
|
|
@ -757,9 +763,12 @@ class FeedExportTest(unittest.TestCase):
|
|||
items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
|
||||
|
||||
formats = {
|
||||
'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),
|
||||
'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'),
|
||||
'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
|
||||
'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
|
||||
'xml': (
|
||||
'<?xml version="1.0" encoding="latin-1"?>\n'
|
||||
'<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'
|
||||
).encode('latin-1'),
|
||||
'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
|
||||
}
|
||||
|
||||
settings = {
|
||||
|
|
|
|||
|
|
@ -1262,7 +1262,10 @@ class XmlRpcRequestTest(RequestTest):
|
|||
class JsonRequestTest(RequestTest):
|
||||
request_class = JsonRequest
|
||||
default_method = 'GET'
|
||||
default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']}
|
||||
default_headers = {
|
||||
b'Content-Type': [b'application/json'],
|
||||
b'Accept': [b'application/json, text/javascript, */*; q=0.01'],
|
||||
}
|
||||
|
||||
def setUp(self):
|
||||
warnings.simplefilter("always")
|
||||
|
|
|
|||
|
|
@ -25,7 +25,11 @@ class BaseResponseTest(unittest.TestCase):
|
|||
self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
|
||||
self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
|
||||
# test presence of all optional parameters
|
||||
self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class))
|
||||
self.assertTrue(
|
||||
isinstance(
|
||||
self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class
|
||||
)
|
||||
)
|
||||
|
||||
r = self.response_class("http://www.example.com")
|
||||
assert isinstance(r.url, str)
|
||||
|
|
@ -323,13 +327,16 @@ class TextResponseTest(BaseResponseTest):
|
|||
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
|
||||
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3",
|
||||
headers={"Content-type": ["text/html; charset=utf-8"]})
|
||||
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3",
|
||||
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
|
||||
|
||||
def test_unicode_body(self):
|
||||
unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'
|
||||
unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
|
||||
'\u0442\u0435\u043a\u0441\u0442')
|
||||
self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
|
||||
|
||||
original_string = unicode_string.encode('cp1251')
|
||||
|
|
@ -344,13 +351,18 @@ class TextResponseTest(BaseResponseTest):
|
|||
self.assertEqual(r1.text, unicode_string)
|
||||
|
||||
def test_encoding(self):
|
||||
r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
|
||||
r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
|
||||
headers={"Content-type": ["text/html; charset=utf-8"]})
|
||||
r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
|
||||
r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=b"\xa3")
|
||||
r3 = self.response_class("http://www.example.com", body=b"\xa3",
|
||||
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
|
||||
r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
|
||||
r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body=b"\xc2\xa3")
|
||||
r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body=b"\xa8D")
|
||||
r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body=b"\xa8D")
|
||||
r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
|
||||
headers={"Content-type": ["text/html; charset=None"]})
|
||||
r6 = self.response_class("http://www.example.com", body=b"\xa8D",
|
||||
headers={"Content-type": ["text/html; charset=gb2312"]})
|
||||
r7 = self.response_class("http://www.example.com", body=b"\xa8D",
|
||||
headers={"Content-type": ["text/html; charset=gbk"]})
|
||||
|
||||
self.assertEqual(r1._headers_encoding(), "utf-8")
|
||||
self.assertEqual(r2._headers_encoding(), None)
|
||||
|
|
@ -685,7 +697,8 @@ class HtmlResponseTest(TextResponseTest):
|
|||
body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
</head><body>Price: \xa3100</body></html>'
|
||||
"""
|
||||
r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body)
|
||||
r3 = self.response_class("http://www.example.com", body=body,
|
||||
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
|
||||
self._assert_response_values(r3, 'iso-8859-1', body)
|
||||
|
||||
# make sure replace() preserves the encoding of the original response
|
||||
|
|
|
|||
|
|
@ -279,8 +279,8 @@ class Base:
|
|||
def test_process_value(self):
|
||||
"""Test restrict_xpaths with encodings"""
|
||||
html = b"""
|
||||
<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
|
||||
<a href="/about.html">About us</a>
|
||||
<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Text</a>
|
||||
<a href="/about.html">About us</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
|
||||
|
||||
|
|
@ -291,7 +291,7 @@ class Base:
|
|||
|
||||
lx = self.extractor_cls(process_value=process_value)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/other/page.html', text='Link text')])
|
||||
[Link(url='http://example.org/other/page.html', text='Text')])
|
||||
|
||||
def test_base_url_with_restrict_xpaths(self):
|
||||
html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
|
||||
|
|
@ -332,7 +332,10 @@ class Base:
|
|||
self.assertEqual(lx.extract_links(self.response), [])
|
||||
|
||||
def test_tags(self):
|
||||
html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
|
||||
html = (
|
||||
b'<html><area href="sample1.html"></area>'
|
||||
b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>'
|
||||
)
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
|
||||
lx = self.extractor_cls(tags=None)
|
||||
|
|
@ -413,24 +416,34 @@ class Base:
|
|||
response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
|
||||
)
|
||||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
|
||||
nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
|
||||
nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
|
||||
)
|
||||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
|
||||
nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
|
||||
nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = b"""
|
||||
|
|
|
|||
|
|
@ -38,27 +38,36 @@ class FilesPipelineTestCase(unittest.TestCase):
|
|||
|
||||
def test_file_path(self):
|
||||
file_path = self.pipeline.file_path
|
||||
self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
|
||||
'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
|
||||
self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
|
||||
'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
|
||||
self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
|
||||
'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
|
||||
self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
|
||||
'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
|
||||
self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
|
||||
'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
|
||||
self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
|
||||
self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
response=Response("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
info=object()),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
|
||||
self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
|
||||
'full/76c00cef2ef669ae65052661f68d451162829507')
|
||||
self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
|
||||
self.assertEqual(
|
||||
file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
|
||||
'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
|
||||
'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
|
||||
self.assertEqual(
|
||||
file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
|
||||
'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
|
||||
'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
|
||||
'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
response=Response("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
info=object()),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
|
||||
'full/76c00cef2ef669ae65052661f68d451162829507')
|
||||
self.assertEqual(
|
||||
file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
|
||||
//+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
|
||||
'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
|
||||
'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
|
||||
|
||||
def test_fs_store(self):
|
||||
assert isinstance(self.pipeline.store, FSFilesStore)
|
||||
|
|
|
|||
|
|
@ -41,22 +41,29 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
|
||||
def test_file_path(self):
|
||||
file_path = self.pipeline.file_path
|
||||
self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.gif")),
|
||||
'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
|
||||
self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
|
||||
'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
|
||||
self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
|
||||
'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
|
||||
self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
|
||||
'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
|
||||
self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
|
||||
'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
|
||||
self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
|
||||
self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
response=Response("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
info=object()),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("https://dev.mydeco.com/mydeco.gif")),
|
||||
'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
|
||||
'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
|
||||
'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
|
||||
'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
|
||||
'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
|
||||
self.assertEqual(
|
||||
file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
response=Response("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
info=object()),
|
||||
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
|
||||
|
||||
def test_thumbnail_name(self):
|
||||
thumb_path = self.pipeline.thumb_path
|
||||
|
|
|
|||
|
|
@ -158,6 +158,12 @@ class CallbackKeywordArgumentsTestCase(TestCase):
|
|||
if key in line.getMessage():
|
||||
exceptions[key] = line
|
||||
self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
|
||||
self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
|
||||
self.assertEqual(
|
||||
str(exceptions['takes_less'].exc_info[1]),
|
||||
"parse_takes_less() got an unexpected keyword argument 'number'"
|
||||
)
|
||||
self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
|
||||
self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")
|
||||
self.assertEqual(
|
||||
str(exceptions['takes_more'].exc_info[1]),
|
||||
"parse_takes_more() missing 1 required positional argument: 'other'"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
def test_from_headers(self):
|
||||
mappings = [
|
||||
({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),
|
||||
({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
|
||||
({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response),
|
||||
({'Content-Type': ['application/octet-stream'],
|
||||
'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
|
||||
]
|
||||
for source, cls in mappings:
|
||||
source = Headers(source)
|
||||
|
|
@ -77,8 +78,10 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
mappings = [
|
||||
({'url': 'http://www.example.com/data.csv'}, TextResponse),
|
||||
# headers takes precedence over url
|
||||
({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse),
|
||||
({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response),
|
||||
({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}),
|
||||
'url': 'http://www.example.com/item/'}, HtmlResponse),
|
||||
({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}),
|
||||
'url': 'http://www.example.com/page/'}, Response),
|
||||
|
||||
|
||||
]
|
||||
|
|
|
|||
|
|
@ -19,18 +19,26 @@ class SelectorTestCase(unittest.TestCase):
|
|||
for x in xl:
|
||||
assert isinstance(x, Selector)
|
||||
|
||||
self.assertEqual(sel.xpath('//input').getall(),
|
||||
[x.get() for x in sel.xpath('//input')])
|
||||
|
||||
self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
|
||||
[u'a'])
|
||||
self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
[u'12.0'])
|
||||
|
||||
self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
|
||||
[u'xpathrules'])
|
||||
self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12'])
|
||||
self.assertEqual(
|
||||
sel.xpath('//input').getall(),
|
||||
[x.get() for x in sel.xpath('//input')]
|
||||
)
|
||||
self.assertEqual(
|
||||
[x.get() for x in sel.xpath("//input[@name='a']/@name")],
|
||||
[u'a']
|
||||
)
|
||||
self.assertEqual(
|
||||
[x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
[u'12.0']
|
||||
)
|
||||
self.assertEqual(
|
||||
sel.xpath("concat('xpath', 'rules')").getall(),
|
||||
[u'xpathrules']
|
||||
)
|
||||
self.assertEqual(
|
||||
[x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12']
|
||||
)
|
||||
|
||||
def test_root_base_url(self):
|
||||
body = b'<html><form action="/path"><input name="a" /></form></html>'
|
||||
|
|
|
|||
|
|
@ -120,7 +120,9 @@ class XMLFeedSpiderTest(SpiderTest):
|
|||
body = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84"
|
||||
xmlns:y="http://www.example.com/schemas/extras/1.0">
|
||||
<url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated><other value="bar" y:custom="fuu"/></url>
|
||||
<url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated>
|
||||
<other value="bar" y:custom="fuu"/>
|
||||
</url>
|
||||
<url><loc>http://www.example.com/</loc><y:updated>2009-08-16</updated><other value="foo"/></url>
|
||||
</urlset>"""
|
||||
response = XmlResponse(url='http://example.com/sitemap.xml', body=body)
|
||||
|
|
|
|||
|
|
@ -385,9 +385,15 @@ class TestSpiderMiddleware(TestCase):
|
|||
log4 = yield self.crawl_log(GeneratorOutputChainSpider)
|
||||
self.assertIn("'item_scraped_count': 2", str(log4))
|
||||
self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4))
|
||||
self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4))
|
||||
self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4))
|
||||
self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4))
|
||||
self.assertIn(
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4))
|
||||
self.assertNotIn(
|
||||
"GeneratorFailMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4))
|
||||
self.assertNotIn(
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4))
|
||||
item_from_callback = {'processed': [
|
||||
'parse-first-item',
|
||||
'GeneratorFailMiddleware.process_spider_output',
|
||||
|
|
@ -414,9 +420,13 @@ class TestSpiderMiddleware(TestCase):
|
|||
log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
|
||||
self.assertIn("'item_scraped_count': 1", str(log5))
|
||||
self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5))
|
||||
self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5))
|
||||
self.assertIn(
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5))
|
||||
self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5))
|
||||
self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5))
|
||||
self.assertNotIn(
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5))
|
||||
item_recovered = {'processed': [
|
||||
'NotGeneratorRecoverMiddleware.process_spider_exception',
|
||||
'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
|
||||
|
|
|
|||
|
|
@ -114,12 +114,16 @@ class MixinNoReferrerWhenDowngrade:
|
|||
class MixinSameOrigin:
|
||||
scenarii = [
|
||||
# Same origin (protocol, host, port): send referrer
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
(
|
||||
'http://example.com:8888/page.html',
|
||||
'http://example.com:8888/not-page.html',
|
||||
b'http://example.com:8888/page.html',
|
||||
),
|
||||
|
||||
# Different host: do NOT send referrer
|
||||
('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
|
||||
|
|
@ -139,8 +143,12 @@ class MixinSameOrigin:
|
|||
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
|
||||
('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
|
||||
(
|
||||
'https://user:password@example.com/page.html',
|
||||
'https://example.com/not-page.html',
|
||||
b'https://example.com/page.html',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -179,12 +187,16 @@ class MixinStrictOrigin:
|
|||
class MixinOriginWhenCrossOrigin:
|
||||
scenarii = [
|
||||
# Same origin (protocol, host, port): send referrer
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
(
|
||||
'http://example.com:8888/page.html',
|
||||
'http://example.com:8888/not-page.html',
|
||||
b'http://example.com:8888/page.html',
|
||||
),
|
||||
|
||||
# Different host: send origin as referrer
|
||||
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
|
||||
|
|
@ -205,21 +217,33 @@ class MixinOriginWhenCrossOrigin:
|
|||
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
|
||||
(
|
||||
'https://user:password@example5.com/page.html',
|
||||
'https://example5.com/not-page.html',
|
||||
b'https://example5.com/page.html',
|
||||
),
|
||||
# TLS to non-TLS downgrade: send origin
|
||||
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),
|
||||
(
|
||||
'https://user:password@example5.com/page.html',
|
||||
'http://example5.com/not-page.html',
|
||||
b'https://example5.com/',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class MixinStrictOriginWhenCrossOrigin:
|
||||
scenarii = [
|
||||
# Same origin (protocol, host, port): send referrer
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
(
|
||||
'http://example.com:8888/page.html',
|
||||
'http://example.com:8888/not-page.html',
|
||||
b'http://example.com:8888/page.html',
|
||||
),
|
||||
|
||||
# Different host: send origin as referrer
|
||||
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
|
||||
|
|
@ -248,7 +272,11 @@ class MixinStrictOriginWhenCrossOrigin:
|
|||
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
|
||||
(
|
||||
'https://user:password@example5.com/page.html',
|
||||
'https://example5.com/not-page.html',
|
||||
b'https://example5.com/page.html',
|
||||
),
|
||||
|
||||
# TLS to non-TLS downgrade: send nothing
|
||||
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None),
|
||||
|
|
@ -281,8 +309,16 @@ class MixinUnsafeUrl:
|
|||
('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
|
||||
|
||||
# test for user/password stripping
|
||||
('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),
|
||||
('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),
|
||||
(
|
||||
'http://user:password@example4.com/page.html',
|
||||
'https://not.example4.com/',
|
||||
b'http://example4.com/page.html',
|
||||
),
|
||||
(
|
||||
'https://user:password@example4.com/page.html',
|
||||
'http://scrapy.org/',
|
||||
b'https://example4.com/page.html',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ class XmliterTestCase(unittest.TestCase):
|
|||
|
||||
def test_xmliter(self):
|
||||
body = b"""<?xml version="1.0" encoding="UTF-8"?>\
|
||||
<products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\
|
||||
<products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:noNamespaceSchemaLocation="someschmea.xsd">\
|
||||
<product id="001">\
|
||||
<type>Type 1</type>\
|
||||
<name>Name 1</name>\
|
||||
|
|
@ -107,7 +108,10 @@ class XmliterTestCase(unittest.TestCase):
|
|||
(u'27', [u'A'], [u'27'])])
|
||||
|
||||
def test_xmliter_text(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
body = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<products><product>one</product><product>two</product></products>'
|
||||
)
|
||||
|
||||
self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
|
||||
[[u'one'], [u'two']])
|
||||
|
|
@ -139,7 +143,10 @@ class XmliterTestCase(unittest.TestCase):
|
|||
self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
|
||||
self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
|
||||
self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
|
||||
self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
self.assertEqual(
|
||||
node.xpath('g:image_link/text()').getall(),
|
||||
['http://www.mydummycompany.com/images/item1.jpg']
|
||||
)
|
||||
self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
|
||||
self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
|
||||
self.assertEqual(node.xpath('image_link/text()').getall(), [])
|
||||
|
|
@ -147,7 +154,10 @@ class XmliterTestCase(unittest.TestCase):
|
|||
self.assertEqual(node.xpath('price/text()').getall(), [])
|
||||
|
||||
def test_xmliter_exception(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
body = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<products><product>one</product><product>two</product></products>'
|
||||
)
|
||||
|
||||
iter = self.xmliter(body, 'product')
|
||||
next(iter)
|
||||
|
|
@ -160,7 +170,12 @@ class XmliterTestCase(unittest.TestCase):
|
|||
self.assertRaises(AssertionError, next, i)
|
||||
|
||||
def test_xmliter_encoding(self):
|
||||
body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
|
||||
body = (
|
||||
b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
|
||||
b'<xml>\n'
|
||||
b' <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n'
|
||||
b'</xml>\n\n'
|
||||
)
|
||||
response = XmlResponse('http://www.example.com', body=body)
|
||||
self.assertEqual(
|
||||
next(self.xmliter(response, 'item')).get(),
|
||||
|
|
|
|||
|
|
@ -75,8 +75,12 @@ class UtilsRequestTest(unittest.TestCase):
|
|||
r1 = Request("http://www.example.com/some/page.html?arg=1")
|
||||
self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
|
||||
|
||||
r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body")
|
||||
self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
|
||||
r1 = Request("http://www.example.com", method='POST',
|
||||
headers={"Content-type": b"text/html"}, body=b"Some body")
|
||||
self.assertEqual(
|
||||
request_httprepr(r1),
|
||||
b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
|
||||
)
|
||||
|
||||
def test_request_httprepr_for_non_http_request(self):
|
||||
# the representation is not important but it must not fail.
|
||||
|
|
|
|||
|
|
@ -22,8 +22,14 @@ class SitemapTest(unittest.TestCase):
|
|||
</url>
|
||||
</urlset>""")
|
||||
assert s.type == 'urlset'
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html',
|
||||
'lastmod': '2009-08-16', 'changefreq': 'weekly'},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_index(self):
|
||||
s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -38,7 +44,13 @@ class SitemapTest(unittest.TestCase):
|
|||
</sitemap>
|
||||
</sitemapindex>""")
|
||||
assert s.type == 'sitemapindex'
|
||||
self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'},
|
||||
{'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_strip(self):
|
||||
"""Assert we can deal with trailing spaces inside <loc> tags - we've
|
||||
|
|
@ -195,11 +207,19 @@ Disallow: /forum/active/
|
|||
</url>
|
||||
</urlset>""")
|
||||
|
||||
self.assertEqual(list(s), [
|
||||
{'loc': 'http://www.example.com/english/',
|
||||
'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
|
||||
}
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{
|
||||
'loc': 'http://www.example.com/english/',
|
||||
'alternate': [
|
||||
'http://www.example.com/deutsch/',
|
||||
'http://www.example.com/schweiz-deutsch/',
|
||||
'http://www.example.com/english/',
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
def test_xml_entity_expansion(self):
|
||||
s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080']))
|
||||
self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15']))
|
||||
|
||||
url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29'
|
||||
url = (
|
||||
'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20'
|
||||
'javascript:%20document.orderform_2581_1190810811.submit%28%29'
|
||||
)
|
||||
self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
|
||||
self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com']))
|
||||
|
||||
|
|
@ -105,8 +108,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
|
|||
'http://username:password@www.example.com')
|
||||
|
||||
def test_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
|
||||
)
|
||||
|
||||
def test_preserve_http(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
|
||||
|
|
@ -137,8 +142,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
|
|||
'http://username:password@www.example.com')
|
||||
|
||||
def test_preserve_http_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
|
||||
)
|
||||
|
||||
def test_protocol_relative(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com'),
|
||||
|
|
@ -169,8 +176,10 @@ class AddHttpIfNoScheme(unittest.TestCase):
|
|||
'http://username:password@www.example.com')
|
||||
|
||||
def test_protocol_relative_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
|
||||
)
|
||||
|
||||
def test_preserve_https(self):
|
||||
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
|
||||
|
|
|
|||
Loading…
Reference in New Issue