diff --git a/pytest.ini b/pytest.ini index 8177fedd3..0e289866c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,227 +20,25 @@ addopts = twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed +flake8-max-line-length = 119 flake8-ignore = W503 - # Files that are only meant to provide top-level imports are expected not - # to use any of their imports: + + # Exclude files that are meant to provide top-level imports + # E402: Module level import not at top of file + # F401: Module imported but unused scrapy/core/downloader/handlers/http.py F401 scrapy/http/__init__.py F401 + scrapy/linkextractors/__init__.py E402 F401 + scrapy/spiders/__init__.py E402 F401 + # Issues pending a review: - # extras - extras/qps-bench-server.py E501 - extras/qpsclient.py E501 E501 - # scrapy/commands - scrapy/commands/__init__.py E501 - scrapy/commands/check.py E501 - scrapy/commands/crawl.py E501 - scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E501 - scrapy/commands/genspider.py E501 - scrapy/commands/parse.py E501 - scrapy/commands/runspider.py E501 - scrapy/commands/settings.py E501 - scrapy/commands/shell.py E501 - scrapy/commands/startproject.py E501 - scrapy/commands/version.py E501 - # scrapy/contracts - scrapy/contracts/__init__.py E501 - # scrapy/core - scrapy/core/engine.py E501 - scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 - scrapy/core/spidermw.py E501 - scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 - scrapy/core/downloader/middleware.py E501 - scrapy/core/downloader/tls.py E501 - scrapy/core/downloader/webclient.py E501 - scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 - scrapy/core/downloader/handlers/http10.py E501 - scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 - # scrapy/downloadermiddlewares - scrapy/downloadermiddlewares/ajaxcrawl.py E501 - scrapy/downloadermiddlewares/decompression.py E501 - scrapy/downloadermiddlewares/defaultheaders.py E501 - scrapy/downloadermiddlewares/httpcache.py E501 - scrapy/downloadermiddlewares/httpcompression.py E501 - scrapy/downloadermiddlewares/httpproxy.py E501 - scrapy/downloadermiddlewares/redirect.py E501 - scrapy/downloadermiddlewares/retry.py E501 - scrapy/downloadermiddlewares/robotstxt.py E501 - scrapy/downloadermiddlewares/stats.py E501 - # scrapy/extensions - scrapy/extensions/closespider.py E501 - scrapy/extensions/corestats.py E501 - scrapy/extensions/feedexport.py E501 - scrapy/extensions/httpcache.py E501 - scrapy/extensions/memdebug.py E501 - scrapy/extensions/spiderstate.py E501 - scrapy/extensions/telnet.py E501 - scrapy/extensions/throttle.py E501 - # scrapy/http - scrapy/http/common.py E501 - scrapy/http/cookies.py E501 - scrapy/http/request/__init__.py E501 - scrapy/http/request/form.py E501 - scrapy/http/request/json_request.py E501 - scrapy/http/response/__init__.py E501 - scrapy/http/response/text.py E501 - # scrapy/linkextractors - scrapy/linkextractors/__init__.py E501 E402 - scrapy/linkextractors/lxmlhtml.py E501 - # scrapy/loader - scrapy/loader/__init__.py E501 - scrapy/loader/processors.py E501 - # scrapy/pipelines - scrapy/pipelines/__init__.py E501 - scrapy/pipelines/files.py E501 - scrapy/pipelines/images.py E501 - scrapy/pipelines/media.py E501 - # scrapy/selector + scrapy/__init__.py E402 scrapy/selector/__init__.py F403 - scrapy/selector/unified.py E501 - # scrapy/settings - scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 - scrapy/settings/deprecated.py E501 - # scrapy/spidermiddlewares - scrapy/spidermiddlewares/httperror.py E501 - scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 - scrapy/spidermiddlewares/urllength.py E501 - # scrapy/spiders - scrapy/spiders/__init__.py E501 E402 - scrapy/spiders/crawl.py E501 - scrapy/spiders/feed.py E501 - scrapy/spiders/sitemap.py E501 - # scrapy/utils - scrapy/utils/asyncio.py E501 - scrapy/utils/benchserver.py E501 - scrapy/utils/conf.py E402 E501 - scrapy/utils/datatypes.py E501 - scrapy/utils/decorators.py E501 - scrapy/utils/defer.py E501 - scrapy/utils/deprecate.py E501 - scrapy/utils/gz.py E501 + scrapy/spiders/__init__.py E402 scrapy/utils/http.py F403 - scrapy/utils/httpobj.py E501 - scrapy/utils/iterators.py E501 - scrapy/utils/log.py E501 scrapy/utils/markup.py F403 - scrapy/utils/misc.py E501 scrapy/utils/multipart.py F403 - scrapy/utils/project.py E501 - scrapy/utils/python.py E501 - scrapy/utils/reactor.py E501 - scrapy/utils/reqser.py E501 - scrapy/utils/request.py E501 - scrapy/utils/response.py E501 - scrapy/utils/signal.py E501 - scrapy/utils/sitemap.py E501 - scrapy/utils/spider.py E501 - scrapy/utils/ssl.py E501 - scrapy/utils/test.py E501 - scrapy/utils/url.py E501 F403 F405 - # scrapy - scrapy/__init__.py E402 E501 - scrapy/cmdline.py E501 - scrapy/crawler.py E501 - scrapy/dupefilters.py E501 - scrapy/exceptions.py E501 - scrapy/exporters.py E501 - scrapy/interfaces.py E501 - scrapy/item.py E501 - scrapy/link.py E501 - scrapy/logformatter.py E501 - scrapy/mail.py E402 E501 - scrapy/middleware.py E501 - scrapy/pqueues.py E501 - scrapy/resolver.py E501 - scrapy/responsetypes.py E501 - scrapy/robotstxt.py E501 - scrapy/shell.py E501 - scrapy/signalmanager.py E501 - scrapy/spiderloader.py E501 - scrapy/squeues.py E501 - scrapy/statscollectors.py E501 - # tests - tests/__init__.py E402 E501 - tests/mockserver.py E501 - tests/spiders.py E501 - tests/test_closespider.py E501 - tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 - tests/test_command_shell.py E501 - tests/test_commands.py E501 - tests/test_contracts.py E501 - tests/test_crawl.py E501 - tests/test_crawler.py E501 - tests/test_dependencies.py E501 - tests/test_downloader_handlers.py E501 - tests/test_downloadermiddleware.py E501 - tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E501 - tests/test_downloadermiddleware_defaultheaders.py E501 - tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 - tests/test_downloadermiddleware_decompression.py E501 - tests/test_downloadermiddleware_httpproxy.py E501 - tests/test_downloadermiddleware_redirect.py E501 - tests/test_downloadermiddleware_retry.py E501 - tests/test_downloadermiddleware_robotstxt.py E501 - tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 - tests/test_engine.py E501 - tests/test_exporters.py E501 - tests/test_feedexport.py E501 - tests/test_http_cookies.py E501 - tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 - tests/test_http_response.py E501 - tests/test_item.py E501 - tests/test_link.py E501 - tests/test_linkextractors.py E501 - tests/test_loader.py E501 E741 - tests/test_logformatter.py E501 - tests/test_mail.py E501 - tests/test_middleware.py E501 - tests/test_pipeline_crawl.py E501 - tests/test_pipeline_files.py E501 - tests/test_pipeline_images.py E501 - tests/test_pipeline_media.py E501 - tests/test_proxy_connect.py E501 - tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 - tests/test_robotstxt_interface.py E501 E501 - tests/test_scheduler.py E501 - tests/test_selector.py E501 - tests/test_spider.py E501 - tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E501 - tests/test_spidermiddleware_offsite.py E501 - tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 - tests/test_squeues.py E501 - tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 - tests/test_utils_curl.py E501 - tests/test_utils_datatypes.py E402 E501 - tests/test_utils_defer.py E501 - tests/test_utils_deprecate.py E501 - tests/test_utils_http.py E501 - tests/test_utils_iterators.py E501 - tests/test_utils_python.py E501 - tests/test_utils_reqser.py E501 - tests/test_utils_request.py E501 - tests/test_utils_response.py E501 - tests/test_utils_sitemap.py E501 - tests/test_utils_url.py E501 E501 - tests/test_webclient.py E501 E402 - tests/test_cmdline/__init__.py E501 - tests/test_settings/__init__.py E501 - tests/test_spiderloader/__init__.py E501 - tests/test_utils_misc/__init__.py E501 + scrapy/utils/url.py F403 F405 + tests/test_loader.py E741 + tests/test_webclient.py E402 diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index a4ec7c8ae..b189e016b 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -165,6 +165,7 @@ if __name__ == '__main__': try: execute() finally: - # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() - # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies + # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit: + # http://doc.pypy.org/en/latest/cpython_differences.html + # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies garbage_collect() diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0a9af3a62..52cf09844 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. - logger.warning("Failure while parsing robots.txt. " - "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.", - exc_info=sys.exc_info(), - extra={'spider': spider}) + logger.warning( + "Failure while parsing robots.txt. File either contains garbage or " + "is in an encoding other than UTF-8, treating it as an empty file.", + exc_info=sys.exc_info(), + extra={'spider': spider}, + ) robotstxt_body = '' return robotstxt_body diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d76a96451..cb021a5a7 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -54,8 +54,12 @@ class Rule: self.process_request = _get_method(self.process_request, spider) self.process_request_argcount = len(get_func_args(self.process_request)) if self.process_request_argcount == 1: - msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' - warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn( + "Rule.process_request should accept two arguments " + "(request, response), accepting only one is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) def _process_request(self, request, response): """ diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 93fda2648..01f164727 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -56,7 +56,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_redirect_not_follow_302(self): - _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + _, out, _ = yield self.execute( + ['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'] + ) assert out.strip().endswith(b'302') @defer.inlineCallbacks diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ecc0cd7af..038fae323 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -305,8 +305,10 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log) self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1e6f744b..51deb20f4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -493,7 +493,10 @@ class Http11TestCase(HttpTestCase): class Https11TestCase(Http11TestCase): scheme = 'https' - tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"' + tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) @defer.inlineCallbacks def test_tls_logging(self): @@ -542,7 +545,10 @@ class Https11InvalidDNSPattern(Https11TestCase): from service_identity.exceptions import CertificateError # noqa: F401 except ImportError: raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) super(Https11InvalidDNSPattern, self).setUp() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index e86568bfb..87304d76c 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -124,7 +124,8 @@ class HttpCompressionTest(TestCase): 'Content-Encoding': 'gzip', } f = BytesIO() - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) zf.close() @@ -142,7 +143,8 @@ class HttpCompressionTest(TestCase): 'Content-Encoding': 'gzip', } f = BytesIO() - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) zf.close() @@ -158,7 +160,8 @@ class HttpCompressionTest(TestCase): headers = { 'Content-Encoding': 'identity', } - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody) response = respcls("http://www.example.com/index", headers=headers, body=plainbody) request = Request("http://www.example.com/index") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 22f23d7b5..c46b1bb87 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -149,7 +149,10 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, 'http://scrapytest.org/redirected') self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') - self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + self.assertEqual( + req3.meta['redirect_urls'], + ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + ) def test_redirect_reasons(self): req1 = Request('http://scrapytest.org/first') @@ -279,7 +282,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, 'http://scrapytest.org/redirected') self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') - self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + self.assertEqual( + req3.meta['redirect_urls'], + ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + ) def test_redirect_reasons(self): req1 = Request('http://scrapytest.org/first') diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 41a8d16bc..95a4fca0d 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -181,7 +181,8 @@ class RFPDupeFilterTest(unittest.TestCase): ( 'scrapy.dupefilters', 'DEBUG', - 'Filtered duplicate request: - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)' + 'Filtered duplicate request: - no more' + ' duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)' ) ) @@ -217,7 +218,8 @@ class RFPDupeFilterTest(unittest.TestCase): ( 'scrapy.dupefilters', 'DEBUG', - 'Filtered duplicate request: (referer: http://scrapytest.org/INDEX.html)' + 'Filtered duplicate request: ' + ' (referer: http://scrapytest.org/INDEX.html)' ) ) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 0f9dafcaa..b27380309 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -111,7 +111,10 @@ class PythonItemExporterTest(BaseItemExporterTest): ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) - self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'} + ) self.assertEqual(type(exported['age']), dict) self.assertEqual(type(exported['age']['age']), dict) @@ -121,7 +124,10 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) @@ -131,7 +137,10 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) @@ -328,13 +337,19 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = b'\n22John\xc2\xa3' + expected_value = ( + b'\n' + b'22John\xc2\xa3' + ) self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - b'\nJohn\xc2\xa3Doe' + ( + b'\n' + b'John\xc2\xa3Doe' + ) ) def test_nested_item(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cbc81bc35..8eeb29b6d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -717,10 +717,13 @@ class FeedExportTest(unittest.TestCase): items = [dict({'foo': u'Test\xd6'})] formats = { - 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), - 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'\nTest\xd6'.encode('utf-8'), - 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), + 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), + 'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'), + 'xml': ( + '\n' + 'Test\xd6' + ).encode('utf-8'), + 'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'), } for fmt, expected in formats.items(): @@ -734,10 +737,13 @@ class FeedExportTest(unittest.TestCase): self.assertEqual(expected, data[fmt]) formats = { - 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), - 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'\nTest\xd6'.encode('latin-1'), - 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), + 'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'), + 'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'), + 'xml': ( + '\n' + 'Test\xd6' + ).encode('latin-1'), + 'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'), } for fmt, expected in formats.items(): @@ -756,9 +762,12 @@ class FeedExportTest(unittest.TestCase): items = [dict({'foo': u'FOO', 'bar': u'BAR'})] formats = { - 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'\n\n \n FOO\n \n'.encode('latin-1'), - 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), + 'xml': ( + '\n' + '\n \n FOO\n \n' + ).encode('latin-1'), + 'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } settings = { diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 349c23f36..63014b22d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1258,7 +1258,10 @@ class XmlRpcRequestTest(RequestTest): class JsonRequestTest(RequestTest): request_class = JsonRequest default_method = 'GET' - default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} + default_headers = { + b'Content-Type': [b'application/json'], + b'Accept': [b'application/json, text/javascript, */*; q=0.01'], + } def setUp(self): warnings.simplefilter("always") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f9b6b92f4..039e863f4 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -26,7 +26,11 @@ class BaseResponseTest(unittest.TestCase): self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) # test presence of all optional parameters - self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class)) + self.assertTrue( + isinstance( + self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class + ) + ) r = self.response_class("http://www.example.com") assert isinstance(r.url, str) @@ -324,13 +328,16 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) + resp = self.response_class(u"http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}) self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) + resp = self.response_class(u"http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') def test_unicode_body(self): - unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442' + unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' + '\u0442\u0435\u043a\u0441\u0442') self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body') original_string = unicode_string.encode('cp1251') @@ -345,13 +352,18 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r1.text, unicode_string) def test_encoding(self): - r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3") + r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}) r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") - r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=b"\xa3") + r3 = self.response_class("http://www.example.com", body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body=b"\xc2\xa3") - r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body=b"\xa8D") - r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body=b"\xa8D") + r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}) + r6 = self.response_class("http://www.example.com", body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}) + r7 = self.response_class("http://www.example.com", body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -697,7 +709,8 @@ class HtmlResponseTest(TextResponseTest): body = b"""Some page Price: \xa3100' """ - r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body) + r3 = self.response_class("http://www.example.com", body=body, + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self._assert_response_values(r3, 'iso-8859-1', body) # make sure replace() preserves the encoding of the original response diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 2a2650480..8d4538eed 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -280,8 +280,8 @@ class Base: def test_process_value(self): """Test restrict_xpaths with encodings""" html = b""" - Link text - About us +Text +About us """ response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') @@ -292,7 +292,7 @@ class Base: lx = self.extractor_cls(process_value=process_value) self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/other/page.html', text='Link text')]) + [Link(url='http://example.org/other/page.html', text='Text')]) def test_base_url_with_restrict_xpaths(self): html = b"""Page title<title><base href="http://otherdomain.com/base/" /> @@ -333,7 +333,10 @@ class Base: self.assertEqual(lx.extract_links(self.response), []) def test_tags(self): - html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>""" + html = ( + b'<html><area href="sample1.html"></area>' + b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>' + ) response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags=None) @@ -419,8 +422,10 @@ class Base: [ Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + fragment='', nofollow=False), Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), ] ) @@ -433,8 +438,10 @@ class Base: [ Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + fragment='', nofollow=False), Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), ] ) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index f155db4ce..34fc8dcb6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -38,27 +38,36 @@ class FilesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') - self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")), - 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') - self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")), - 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), - 'full/76c00cef2ef669ae65052661f68d451162829507') - self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ + self.assertEqual( + file_path(Request("https://dev.mydeco.com/mydeco.pdf")), + 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + self.assertEqual( + file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")), + 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")), + 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object()), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), + 'full/76c00cef2ef669ae65052661f68d451162829507') + self.assertEqual( + file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")), - 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') + 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 76aea470f..8ef27fce7 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -41,22 +41,29 @@ class ImagesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.gif")), - 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') - self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")), - 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg') - self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")), - 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/mydeco.gif")), + 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual( + file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")), + 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")), + 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object()), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') def test_thumbnail_name(self): thumb_path = self.pipeline.thumb_path diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a3ddd50f4..bd49179aa 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -158,6 +158,12 @@ class CallbackKeywordArgumentsTestCase(TestCase): if key in line.getMessage(): exceptions[key] = line self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) - self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'") + self.assertEqual( + str(exceptions['takes_less'].exc_info[1]), + "parse_takes_less() got an unexpected keyword argument 'number'" + ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'") + self.assertEqual( + str(exceptions['takes_more'].exc_info[1]), + "parse_takes_more() missing 1 required positional argument: 'other'" + ) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 9e63ac924..dd19a69d5 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -63,8 +63,9 @@ class ResponseTypesTest(unittest.TestCase): def test_from_headers(self): mappings = [ ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse), - ({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse), ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response), + ({'Content-Type': ['application/octet-stream'], + 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse), ] for source, cls in mappings: source = Headers(source) @@ -76,8 +77,10 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ ({'url': 'http://www.example.com/data.csv'}, TextResponse), # headers takes precedence over url - ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse), - ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response), + ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), + 'url': 'http://www.example.com/item/'}, HtmlResponse), + ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), + 'url': 'http://www.example.com/page/'}, Response), ] diff --git a/tests/test_selector.py b/tests/test_selector.py index 65b0f5860..bcf653444 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -19,18 +19,26 @@ class SelectorTestCase(unittest.TestCase): for x in xl: assert isinstance(x, Selector) - self.assertEqual(sel.xpath('//input').getall(), - [x.get() for x in sel.xpath('//input')]) - - self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")], - [u'a']) - self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0']) - - self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), - [u'xpathrules']) - self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12']) + self.assertEqual( + sel.xpath('//input').getall(), + [x.get() for x in sel.xpath('//input')] + ) + self.assertEqual( + [x.get() for x in sel.xpath("//input[@name='a']/@name")], + [u'a'] + ) + self.assertEqual( + [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + [u'12.0'] + ) + self.assertEqual( + sel.xpath("concat('xpath', 'rules')").getall(), + [u'xpathrules'] + ) + self.assertEqual( + [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + [u'12'] + ) def test_root_base_url(self): body = b'<html><form action="/path"><input name="a" /></form></html>' diff --git a/tests/test_spider.py b/tests/test_spider.py index bb00c8f42..805d70459 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -120,7 +120,9 @@ class XMLFeedSpiderTest(SpiderTest): body = b"""<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84" xmlns:y="http://www.example.com/schemas/extras/1.0"> - <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated><other value="bar" y:custom="fuu"/></url> + <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated> + <other value="bar" y:custom="fuu"/> + </url> <url><loc>http://www.example.com/</loc><y:updated>2009-08-16</updated><other value="foo"/></url> </urlset>""" response = XmlResponse(url='http://example.com/sitemap.xml', body=body) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index ad4d6fb98..79eda35b3 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -385,9 +385,15 @@ class TestSpiderMiddleware(TestCase): log4 = yield self.crawl_log(GeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 2", str(log4)) self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", + str(log4)) + self.assertNotIn( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught", + str(log4)) + self.assertNotIn( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", + str(log4)) item_from_callback = {'processed': [ 'parse-first-item', 'GeneratorFailMiddleware.process_spider_output', @@ -414,9 +420,13 @@ class TestSpiderMiddleware(TestCase): log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 1", str(log5)) self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5)) - self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertIn( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", + str(log5)) self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5)) - self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertNotIn( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", + str(log5)) item_recovered = {'processed': [ 'NotGeneratorRecoverMiddleware.process_spider_exception', 'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index ca765518b..067118cf0 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -119,7 +119,11 @@ class MixinSameOrigin: ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: do NOT send referrer ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), @@ -139,8 +143,12 @@ class MixinSameOrigin: ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ( + 'https://user:password@example.com/page.html', + 'https://example.com/not-page.html', + b'https://example.com/page.html', + ), ] @@ -184,7 +192,11 @@ class MixinOriginWhenCrossOrigin: ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: send origin as referrer ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), @@ -205,9 +217,17 @@ class MixinOriginWhenCrossOrigin: ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + ( + 'https://user:password@example5.com/page.html', + 'https://example5.com/not-page.html', + b'https://example5.com/page.html', + ), # TLS to non-TLS downgrade: send origin - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ( + 'https://user:password@example5.com/page.html', + 'http://example5.com/not-page.html', + b'https://example5.com/', + ), ] @@ -219,7 +239,11 @@ class MixinStrictOriginWhenCrossOrigin: ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: send origin as referrer ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), @@ -248,7 +272,11 @@ class MixinStrictOriginWhenCrossOrigin: ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + ( + 'https://user:password@example5.com/page.html', + 'https://example5.com/not-page.html', + b'https://example5.com/page.html', + ), # TLS to non-TLS downgrade: send nothing ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), @@ -281,8 +309,16 @@ class MixinUnsafeUrl: ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), # test for user/password stripping - ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), - ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ( + 'http://user:password@example4.com/page.html', + 'https://not.example4.com/', + b'http://example4.com/page.html', + ), + ( + 'https://user:password@example4.com/page.html', + 'http://scrapy.org/', + b'https://example4.com/page.html', + ), ] diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 824170d71..8344c6701 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -16,7 +16,8 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter(self): body = b"""<?xml version="1.0" encoding="UTF-8"?>\ - <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\ + <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="someschmea.xsd">\ <product id="001">\ <type>Type 1</type>\ <name>Name 1</name>\ @@ -105,7 +106,10 @@ class XmliterTestCase(unittest.TestCase): (u'27', [u'A'], [u'27'])]) def test_xmliter_text(self): - body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" + body = ( + '<?xml version="1.0" encoding="UTF-8"?>' + '<products><product>one</product><product>two</product></products>' + ) self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) @@ -137,7 +141,10 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('title/text()').getall(), ['Item 1']) self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1']) self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual( + node.xpath('g:image_link/text()').getall(), + ['http://www.mydummycompany.com/images/item1.jpg'] + ) self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1']) self.assertEqual(node.xpath('g:price/text()').getall(), ['400']) self.assertEqual(node.xpath('image_link/text()').getall(), []) @@ -145,7 +152,10 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('price/text()').getall(), []) def test_xmliter_exception(self): - body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" + body = ( + '<?xml version="1.0" encoding="UTF-8"?>' + '<products><product>one</product><product>two</product></products>' + ) iter = self.xmliter(body, 'product') next(iter) @@ -158,7 +168,12 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(TypeError, next, i) def test_xmliter_encoding(self): - body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n' + body = ( + b'<?xml version="1.0" encoding="ISO-8859-9"?>\n' + b'<xml>\n' + b' <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n' + b'</xml>\n\n' + ) response = XmlResponse('http://www.example.com', body=body) self.assertEqual( next(self.xmliter(response, 'item')).get(), diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 50efb63ca..4cd4b7010 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -76,8 +76,12 @@ class UtilsRequestTest(unittest.TestCase): r1 = Request("http://www.example.com/some/page.html?arg=1") self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n') - r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body") - self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Request("http://www.example.com", method='POST', + headers={"Content-type": b"text/html"}, body=b"Some body") + self.assertEqual( + request_httprepr(r1), + b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body' + ) def test_request_httprepr_for_non_http_request(self): # the representation is not important but it must not fail. diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index d77978ff1..bfbf9abb3 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -26,7 +26,8 @@ class SitemapTest(unittest.TestCase): list(s), [ {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, + {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', + 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, ] ) @@ -43,7 +44,13 @@ class SitemapTest(unittest.TestCase): </sitemap> </sitemapindex>""") assert s.type == 'sitemapindex' - self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}]) + self.assertEqual( + list(s), + [ + {'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, + {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}, + ] + ) def test_sitemap_strip(self): """Assert we can deal with trailing spaces inside <loc> tags - we've diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 4ab944949..09a6d6c70 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -27,7 +27,10 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) - url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' + url = ( + 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20' + 'javascript:%20document.orderform_2581_1190810811.submit%28%29' + ) self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com']))