From 182394bcecf556854291f1a0d2e0d2c406bc7b48 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 22 Mar 2020 23:29:30 -0300 Subject: [PATCH] E501 compliance (tests) --- pytest.ini | 129 ++++++++---------- tests/test_command_shell.py | 4 +- tests/test_crawler.py | 4 +- tests/test_downloader_handlers.py | 10 +- ...st_downloadermiddleware_httpcompression.py | 9 +- tests/test_downloadermiddleware_redirect.py | 10 +- tests/test_exporters.py | 25 +++- tests/test_feedexport.py | 31 +++-- tests/test_http_request.py | 5 +- tests/test_http_response.py | 33 +++-- tests/test_linkextractors.py | 49 ++++--- tests/test_pipeline_files.py | 49 ++++--- tests/test_pipeline_images.py | 39 +++--- tests/test_request_cb_kwargs.py | 10 +- tests/test_responsetypes.py | 9 +- tests/test_selector.py | 32 +++-- tests/test_spider.py | 4 +- tests/test_spidermiddleware_output_chain.py | 20 ++- tests/test_spidermiddleware_referer.py | 86 ++++++++---- tests/test_utils_iterators.py | 25 +++- tests/test_utils_request.py | 8 +- tests/test_utils_sitemap.py | 36 +++-- tests/test_utils_url.py | 23 +++- 23 files changed, 415 insertions(+), 235 deletions(-) diff --git a/pytest.ini b/pytest.ini index 781a45541..6f7e09478 100644 --- a/pytest.ini +++ b/pytest.ini @@ -108,84 +108,63 @@ flake8-ignore = scrapy/spiderloader.py F841 E126 scrapy/squeues.py E128 # tests - tests/__init__.py E402 E501 - tests/mockserver.py E401 E501 E126 E123 + tests/__init__.py E402 + tests/mockserver.py E401 E126 E123 tests/pipelines.py F841 - tests/spiders.py E501 E127 - tests/test_closespider.py E501 E127 - tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 - tests/test_command_shell.py E501 E128 - tests/test_commands.py E128 E501 - tests/test_contracts.py E501 E128 - tests/test_crawl.py E501 E741 E265 - tests/test_crawler.py F841 E501 - tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123 - tests/test_downloadermiddleware.py E501 - tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126 + tests/spiders.py E127 + tests/test_closespider.py E127 + tests/test_command_parse.py E128 + tests/test_command_shell.py E128 + tests/test_commands.py E128 + tests/test_contracts.py E128 + tests/test_crawl.py E741 E265 + tests/test_crawler.py F841 + tests/test_dependencies.py F841 + tests/test_downloader_handlers.py E124 E127 E128 E265 E126 E123 + tests/test_downloadermiddleware_cookies.py E731 E741 E128 E265 E126 tests/test_downloadermiddleware_decompression.py E127 - tests/test_downloadermiddleware_defaultheaders.py E501 - tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 E126 E123 - tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E128 E127 - tests/test_downloadermiddleware_retry.py E501 E128 E126 - tests/test_downloadermiddleware_robotstxt.py E501 - tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 E124 - tests/test_engine.py E401 E501 E128 - tests/test_exporters.py E501 E731 E128 E124 + tests/test_downloadermiddleware_httpcompression.py E126 E123 + tests/test_downloadermiddleware_httpproxy.py E128 + tests/test_downloadermiddleware_redirect.py E128 E127 + tests/test_downloadermiddleware_retry.py E128 E126 + tests/test_dupefilters.py E741 E128 E124 + tests/test_engine.py E401 E128 + tests/test_exporters.py E731 E128 E124 tests/test_extension_telnet.py F841 - tests/test_feedexport.py E501 F841 E241 - tests/test_http_cookies.py E501 - tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 - tests/test_http_response.py E501 E128 E265 + tests/test_feedexport.py F841 E241 + tests/test_http_request.py E402 E127 E128 E128 E126 E123 + tests/test_http_response.py E128 E265 tests/test_item.py E128 F841 - tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E501 E731 E741 E128 E117 E241 - tests/test_logformatter.py E128 E501 E122 - tests/test_mail.py E128 E501 - tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 E126 - tests/test_pipeline_files.py E501 - tests/test_pipeline_images.py F841 E501 - tests/test_pipeline_media.py E501 E741 E731 E128 E502 - tests/test_proxy_connect.py E501 E741 - tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 - tests/test_robotstxt_interface.py E501 E501 - tests/test_scheduler.py E501 E126 E123 - tests/test_selector.py E501 E127 - tests/test_spider.py E501 - tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 - tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 - tests/test_squeues.py E501 E741 - tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 E128 - tests/test_utils_curl.py E501 - tests/test_utils_datatypes.py E402 E501 - tests/test_utils_defer.py E501 F841 - tests/test_utils_deprecate.py F841 E501 - tests/test_utils_http.py E501 E128 W504 - tests/test_utils_iterators.py E501 E128 E129 E241 + tests/test_linkextractors.py E128 E124 + tests/test_loader.py E731 E741 E128 E117 E241 + tests/test_logformatter.py E128 E122 + tests/test_mail.py E128 + tests/test_middleware.py E128 + tests/test_pipeline_crawl.py E128 E126 + tests/test_pipeline_images.py F841 + tests/test_pipeline_media.py E741 E731 E128 E502 + tests/test_proxy_connect.py E741 + tests/test_scheduler.py E126 E123 + tests/test_selector.py E127 + tests/test_spidermiddleware_httperror.py E128 E127 E121 + tests/test_spidermiddleware_offsite.py E128 E111 + tests/test_spidermiddleware_referer.py F841 E125 E201 E124 E241 E121 + tests/test_squeues.py E741 + tests/test_utils_conf.py E128 + tests/test_utils_datatypes.py E402 + tests/test_utils_defer.py F841 + tests/test_utils_deprecate.py F841 + tests/test_utils_http.py E128 W504 + tests/test_utils_iterators.py E128 E129 E241 tests/test_utils_log.py E741 - tests/test_utils_python.py E501 E731 - tests/test_utils_reqser.py E501 E128 - tests/test_utils_request.py E501 E128 - tests/test_utils_response.py E501 + tests/test_utils_python.py E731 + tests/test_utils_reqser.py E128 + tests/test_utils_request.py E128 tests/test_utils_signal.py E741 F841 E731 - tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123 - tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126 - tests/test_cmdline/__init__.py E501 - tests/test_settings/__init__.py E501 E128 - tests/test_spiderloader/__init__.py E128 E501 - tests/test_utils_misc/__init__.py E501 + tests/test_utils_sitemap.py E128 E124 + tests/test_utils_url.py E127 E125 E241 E126 E123 + tests/test_webclient.py E128 E122 E402 E241 E123 E126 + tests/test_cmdline/__init__.py + tests/test_settings/__init__.py E128 + tests/test_spiderloader/__init__.py E128 + tests/test_utils_misc/__init__.py diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..5dc0ba3d2 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -56,7 +56,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_redirect_not_follow_302(self): - _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + _, out, _ = yield self.execute( + ['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'] + ) assert out.strip().endswith(b'302') @defer.inlineCallbacks diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 169e763f0..bfc9a609a 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -303,8 +303,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log) self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 29d06bab4..1a7be5c1b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -493,7 +493,10 @@ class Http11TestCase(HttpTestCase): class Https11TestCase(Http11TestCase): scheme = 'https' - tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"' + tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) @defer.inlineCallbacks def test_tls_logging(self): @@ -542,7 +545,10 @@ class Https11InvalidDNSPattern(Https11TestCase): from service_identity.exceptions import CertificateError # noqa: F401 except ImportError: raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) super(Https11InvalidDNSPattern, self).setUp() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 106ca3360..4d43e164f 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -124,7 +124,8 @@ class HttpCompressionTest(TestCase): 'Content-Encoding': 'gzip', } f = BytesIO() - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) zf.close() @@ -142,7 +143,8 @@ class HttpCompressionTest(TestCase): 'Content-Encoding': 'gzip', } f = BytesIO() - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) zf.close() @@ -158,7 +160,8 @@ class HttpCompressionTest(TestCase): headers = { 'Content-Encoding': 'identity', } - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody) response = respcls("http://www.example.com/index", headers=headers, body=plainbody) request = Request("http://www.example.com/index") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 053e26fc3..3044c5b21 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -151,7 +151,10 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, 'http://scrapytest.org/redirected') self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') - self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + self.assertEqual( + req3.meta['redirect_urls'], + ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + ) def test_redirect_reasons(self): req1 = Request('http://scrapytest.org/first') @@ -282,7 +285,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, 'http://scrapytest.org/redirected') self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') - self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + self.assertEqual( + req3.meta['redirect_urls'], + ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + ) def test_redirect_reasons(self): req1 = Request('http://scrapytest.org/first') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6e2507508..b8301a567 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -111,7 +111,10 @@ class PythonItemExporterTest(BaseItemExporterTest): ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) - self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'} + ) self.assertEqual(type(exported['age']), dict) self.assertEqual(type(exported['age']['age']), dict) @@ -121,7 +124,10 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) @@ -131,7 +137,10 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) @@ -327,13 +336,19 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = b'\n22John\xc2\xa3' + expected_value = ( + b'\n' + b'22John\xc2\xa3' + ) self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - b'\nJohn\xc2\xa3Doe' + ( + b'\n' + b'John\xc2\xa3Doe' + ) ) def test_nested_item(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c5589e52f..e248969db 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -718,10 +718,13 @@ class FeedExportTest(unittest.TestCase): header = ['foo'] formats = { - 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), - 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'\nTest\xd6'.encode('utf-8'), - 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), + 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), + 'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'), + 'xml': ( + '\n' + 'Test\xd6' + ).encode('utf-8'), + 'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'), } for fmt, expected in formats.items(): @@ -735,10 +738,13 @@ class FeedExportTest(unittest.TestCase): self.assertEqual(expected, data[fmt]) formats = { - 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), - 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'\nTest\xd6'.encode('latin-1'), - 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), + 'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'), + 'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'), + 'xml': ( + '\n' + 'Test\xd6' + ).encode('latin-1'), + 'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'), } for fmt, expected in formats.items(): @@ -757,9 +763,12 @@ class FeedExportTest(unittest.TestCase): items = [dict({'foo': u'FOO', 'bar': u'BAR'})] formats = { - 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'\n\n \n FOO\n \n'.encode('latin-1'), - 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), + 'xml': ( + '\n' + '\n \n FOO\n \n' + ).encode('latin-1'), + 'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } settings = { diff --git a/tests/test_http_request.py b/tests/test_http_request.py index cc2cddda4..87d98d6ee 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1262,7 +1262,10 @@ class XmlRpcRequestTest(RequestTest): class JsonRequestTest(RequestTest): request_class = JsonRequest default_method = 'GET' - default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} + default_headers = { + b'Content-Type': [b'application/json'], + b'Accept': [b'application/json, text/javascript, */*; q=0.01'], + } def setUp(self): warnings.simplefilter("always") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index eafc3560e..a6b717bd1 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -25,7 +25,11 @@ class BaseResponseTest(unittest.TestCase): self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) # test presence of all optional parameters - self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class)) + self.assertTrue( + isinstance( + self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class + ) + ) r = self.response_class("http://www.example.com") assert isinstance(r.url, str) @@ -323,13 +327,16 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) + resp = self.response_class(u"http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}) self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) + resp = self.response_class(u"http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') def test_unicode_body(self): - unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442' + unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' + '\u0442\u0435\u043a\u0441\u0442') self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body') original_string = unicode_string.encode('cp1251') @@ -344,13 +351,18 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r1.text, unicode_string) def test_encoding(self): - r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3") + r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}) r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") - r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=b"\xa3") + r3 = self.response_class("http://www.example.com", body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body=b"\xc2\xa3") - r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body=b"\xa8D") - r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body=b"\xa8D") + r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}) + r6 = self.response_class("http://www.example.com", body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}) + r7 = self.response_class("http://www.example.com", body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -685,7 +697,8 @@ class HtmlResponseTest(TextResponseTest): body = b"""Some page Price: \xa3100' """ - r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body) + r3 = self.response_class("http://www.example.com", body=body, + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self._assert_response_values(r3, 'iso-8859-1', body) # make sure replace() preserves the encoding of the original response diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 53968e60e..ed5f2560a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -279,8 +279,8 @@ class Base: def test_process_value(self): """Test restrict_xpaths with encodings""" html = b""" - Link text - About us +Text +About us """ response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') @@ -291,7 +291,7 @@ class Base: lx = self.extractor_cls(process_value=process_value) self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/other/page.html', text='Link text')]) + [Link(url='http://example.org/other/page.html', text='Text')]) def test_base_url_with_restrict_xpaths(self): html = b"""Page title<title><base href="http://otherdomain.com/base/" /> @@ -332,7 +332,10 @@ class Base: self.assertEqual(lx.extract_links(self.response), []) def test_tags(self): - html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>""" + html = ( + b'<html><area href="sample1.html"></area>' + b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>' + ) response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags=None) @@ -413,24 +416,34 @@ class Base: response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', + nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', + nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) response = XmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', + nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', + nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) def test_link_wrong_href(self): html = b""" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index f155db4ce..34fc8dcb6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -38,27 +38,36 @@ class FilesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') - self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")), - 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') - self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")), - 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), - 'full/76c00cef2ef669ae65052661f68d451162829507') - self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ + self.assertEqual( + file_path(Request("https://dev.mydeco.com/mydeco.pdf")), + 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + self.assertEqual( + file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")), + 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")), + 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object()), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), + 'full/76c00cef2ef669ae65052661f68d451162829507') + self.assertEqual( + file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")), - 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') + 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 5018d6802..e1c3a55dd 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -41,22 +41,29 @@ class ImagesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.gif")), - 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') - self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")), - 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg') - self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")), - 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/mydeco.gif")), + 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual( + file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")), + 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")), + 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object()), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') def test_thumbnail_name(self): thumb_path = self.pipeline.thumb_path diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a3ddd50f4..bd49179aa 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -158,6 +158,12 @@ class CallbackKeywordArgumentsTestCase(TestCase): if key in line.getMessage(): exceptions[key] = line self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) - self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'") + self.assertEqual( + str(exceptions['takes_less'].exc_info[1]), + "parse_takes_less() got an unexpected keyword argument 'number'" + ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'") + self.assertEqual( + str(exceptions['takes_more'].exc_info[1]), + "parse_takes_more() missing 1 required positional argument: 'other'" + ) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 8cdf7a176..9f4d56412 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -64,8 +64,9 @@ class ResponseTypesTest(unittest.TestCase): def test_from_headers(self): mappings = [ ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse), - ({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse), ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response), + ({'Content-Type': ['application/octet-stream'], + 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse), ] for source, cls in mappings: source = Headers(source) @@ -77,8 +78,10 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ ({'url': 'http://www.example.com/data.csv'}, TextResponse), # headers takes precedence over url - ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse), - ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response), + ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), + 'url': 'http://www.example.com/item/'}, HtmlResponse), + ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), + 'url': 'http://www.example.com/page/'}, Response), ] diff --git a/tests/test_selector.py b/tests/test_selector.py index 09c2546fb..3629feb16 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -19,18 +19,26 @@ class SelectorTestCase(unittest.TestCase): for x in xl: assert isinstance(x, Selector) - self.assertEqual(sel.xpath('//input').getall(), - [x.get() for x in sel.xpath('//input')]) - - self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")], - [u'a']) - self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0']) - - self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), - [u'xpathrules']) - self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12']) + self.assertEqual( + sel.xpath('//input').getall(), + [x.get() for x in sel.xpath('//input')] + ) + self.assertEqual( + [x.get() for x in sel.xpath("//input[@name='a']/@name")], + [u'a'] + ) + self.assertEqual( + [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + [u'12.0'] + ) + self.assertEqual( + sel.xpath("concat('xpath', 'rules')").getall(), + [u'xpathrules'] + ) + self.assertEqual( + [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + [u'12'] + ) def test_root_base_url(self): body = b'<html><form action="/path"><input name="a" /></form></html>' diff --git a/tests/test_spider.py b/tests/test_spider.py index bb00c8f42..805d70459 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -120,7 +120,9 @@ class XMLFeedSpiderTest(SpiderTest): body = b"""<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84" xmlns:y="http://www.example.com/schemas/extras/1.0"> - <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated><other value="bar" y:custom="fuu"/></url> + <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated> + <other value="bar" y:custom="fuu"/> + </url> <url><loc>http://www.example.com/</loc><y:updated>2009-08-16</updated><other value="foo"/></url> </urlset>""" response = XmlResponse(url='http://example.com/sitemap.xml', body=body) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index b26353d6c..78d2de347 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -385,9 +385,15 @@ class TestSpiderMiddleware(TestCase): log4 = yield self.crawl_log(GeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 2", str(log4)) self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", + str(log4)) + self.assertNotIn( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught", + str(log4)) + self.assertNotIn( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", + str(log4)) item_from_callback = {'processed': [ 'parse-first-item', 'GeneratorFailMiddleware.process_spider_output', @@ -414,9 +420,13 @@ class TestSpiderMiddleware(TestCase): log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 1", str(log5)) self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5)) - self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertIn( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", + str(log5)) self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5)) - self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertNotIn( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", + str(log5)) item_recovered = {'processed': [ 'NotGeneratorRecoverMiddleware.process_spider_exception', 'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4c6ede70b..3974f7884 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -114,12 +114,16 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: do NOT send referrer ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), @@ -139,8 +143,12 @@ class MixinSameOrigin: ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ( + 'https://user:password@example.com/page.html', + 'https://example.com/not-page.html', + b'https://example.com/page.html', + ), ] @@ -179,12 +187,16 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: send origin as referrer ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), @@ -205,21 +217,33 @@ class MixinOriginWhenCrossOrigin: ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + ( + 'https://user:password@example5.com/page.html', + 'https://example5.com/not-page.html', + b'https://example5.com/page.html', + ), # TLS to non-TLS downgrade: send origin - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ( + 'https://user:password@example5.com/page.html', + 'http://example5.com/not-page.html', + b'https://example5.com/', + ), ] class MixinStrictOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: send origin as referrer ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), @@ -248,7 +272,11 @@ class MixinStrictOriginWhenCrossOrigin: ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + ( + 'https://user:password@example5.com/page.html', + 'https://example5.com/not-page.html', + b'https://example5.com/page.html', + ), # TLS to non-TLS downgrade: send nothing ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), @@ -281,8 +309,16 @@ class MixinUnsafeUrl: ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), # test for user/password stripping - ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), - ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ( + 'http://user:password@example4.com/page.html', + 'https://not.example4.com/', + b'http://example4.com/page.html', + ), + ( + 'https://user:password@example4.com/page.html', + 'http://scrapy.org/', + b'https://example4.com/page.html', + ), ] diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 33fc4d570..7c4011142 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -17,7 +17,8 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter(self): body = b"""<?xml version="1.0" encoding="UTF-8"?>\ - <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\ + <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="someschmea.xsd">\ <product id="001">\ <type>Type 1</type>\ <name>Name 1</name>\ @@ -107,7 +108,10 @@ class XmliterTestCase(unittest.TestCase): (u'27', [u'A'], [u'27'])]) def test_xmliter_text(self): - body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" + body = ( + '<?xml version="1.0" encoding="UTF-8"?>' + '<products><product>one</product><product>two</product></products>' + ) self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) @@ -139,7 +143,10 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('title/text()').getall(), ['Item 1']) self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1']) self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual( + node.xpath('g:image_link/text()').getall(), + ['http://www.mydummycompany.com/images/item1.jpg'] + ) self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1']) self.assertEqual(node.xpath('g:price/text()').getall(), ['400']) self.assertEqual(node.xpath('image_link/text()').getall(), []) @@ -147,7 +154,10 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('price/text()').getall(), []) def test_xmliter_exception(self): - body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" + body = ( + '<?xml version="1.0" encoding="UTF-8"?>' + '<products><product>one</product><product>two</product></products>' + ) iter = self.xmliter(body, 'product') next(iter) @@ -160,7 +170,12 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(AssertionError, next, i) def test_xmliter_encoding(self): - body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n' + body = ( + b'<?xml version="1.0" encoding="ISO-8859-9"?>\n' + b'<xml>\n' + b' <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n' + b'</xml>\n\n' + ) response = XmlResponse('http://www.example.com', body=body) self.assertEqual( next(self.xmliter(response, 'item')).get(), diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 45f0f59e4..3b1994019 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -75,8 +75,12 @@ class UtilsRequestTest(unittest.TestCase): r1 = Request("http://www.example.com/some/page.html?arg=1") self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n') - r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body") - self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Request("http://www.example.com", method='POST', + headers={"Content-type": b"text/html"}, body=b"Some body") + self.assertEqual( + request_httprepr(r1), + b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body' + ) def test_request_httprepr_for_non_http_request(self): # the representation is not important but it must not fail. diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index db323ab31..4c09d33b2 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -22,8 +22,14 @@ class SitemapTest(unittest.TestCase): </url> </urlset>""") assert s.type == 'urlset' - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', + 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, + ] + ) def test_sitemap_index(self): s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?> @@ -38,7 +44,13 @@ class SitemapTest(unittest.TestCase): </sitemap> </sitemapindex>""") assert s.type == 'sitemapindex' - self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}]) + self.assertEqual( + list(s), + [ + {'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, + {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}, + ] + ) def test_sitemap_strip(self): """Assert we can deal with trailing spaces inside <loc> tags - we've @@ -195,11 +207,19 @@ Disallow: /forum/active/ </url> </urlset>""") - self.assertEqual(list(s), [ - {'loc': 'http://www.example.com/english/', - 'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/'] - } - ]) + self.assertEqual( + list(s), + [ + { + 'loc': 'http://www.example.com/english/', + 'alternate': [ + 'http://www.example.com/deutsch/', + 'http://www.example.com/schweiz-deutsch/', + 'http://www.example.com/english/', + ], + } + ] + ) def test_xml_entity_expansion(self): s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?> diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 7abff8281..a12b6dda2 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -28,7 +28,10 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) - url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' + url = ( + 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20' + 'javascript:%20document.orderform_2581_1190810811.submit%28%29' + ) self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com'])) @@ -105,8 +108,10 @@ class AddHttpIfNoScheme(unittest.TestCase): 'http://username:password@www.example.com') def test_complete_url(self): - self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' + ) def test_preserve_http(self): self.assertEqual(add_http_if_no_scheme('http://www.example.com'), @@ -137,8 +142,10 @@ class AddHttpIfNoScheme(unittest.TestCase): 'http://username:password@www.example.com') def test_preserve_http_complete_url(self): - self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' + ) def test_protocol_relative(self): self.assertEqual(add_http_if_no_scheme('//www.example.com'), @@ -169,8 +176,10 @@ class AddHttpIfNoScheme(unittest.TestCase): 'http://username:password@www.example.com') def test_protocol_relative_complete_url(self): - self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' + ) def test_preserve_https(self): self.assertEqual(add_http_if_no_scheme('https://www.example.com'),