From 36a3913a41a033e57c6119edd8bbf967044c927f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 22 Mar 2020 21:34:20 -0300 Subject: [PATCH 01/20] E501 compliance --- pytest.ini | 176 +++++++++------------------ scrapy/cmdline.py | 5 +- scrapy/core/downloader/middleware.py | 18 ++- scrapy/robotstxt.py | 10 +- scrapy/spiders/crawl.py | 8 +- 5 files changed, 85 insertions(+), 132 deletions(-) diff --git a/pytest.ini b/pytest.ini index 141a13a4f..781a45541 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,6 +20,7 @@ addopts = twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed +flake8-max-line-length = 119 flake8-ignore = W503 # Files that are only meant to provide top-level imports are expected not @@ -27,146 +28,85 @@ flake8-ignore = scrapy/core/downloader/handlers/http.py F401 scrapy/http/__init__.py F401 # Issues pending a review: - # extras - extras/qps-bench-server.py E501 - extras/qpsclient.py E501 E501 # scrapy/commands - scrapy/commands/__init__.py E128 E501 - scrapy/commands/check.py E501 - scrapy/commands/crawl.py E501 - scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E501 E128 E731 - scrapy/commands/genspider.py E128 E501 E502 - scrapy/commands/parse.py E128 E501 E731 - scrapy/commands/runspider.py E501 + scrapy/commands/__init__.py E128 + scrapy/commands/fetch.py E401 E128 E731 + scrapy/commands/genspider.py E128 E502 + scrapy/commands/parse.py E128 E731 + scrapy/commands/runspider.py scrapy/commands/settings.py E128 - scrapy/commands/shell.py E128 E501 E502 - scrapy/commands/startproject.py E127 E501 E128 - scrapy/commands/version.py E501 E128 + scrapy/commands/shell.py E128 E502 + scrapy/commands/startproject.py E127 E128 + scrapy/commands/version.py E128 # scrapy/contracts - scrapy/contracts/__init__.py E501 W504 + scrapy/contracts/__init__.py W504 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 E127 E502 - scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 W504 - scrapy/core/spidermw.py E501 E731 E126 - scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 E128 E126 - scrapy/core/downloader/middleware.py E501 E502 - scrapy/core/downloader/tls.py E501 E241 - scrapy/core/downloader/webclient.py E731 E501 E128 E126 - scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E128 E127 - scrapy/core/downloader/handlers/http10.py E501 - scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E128 E126 + scrapy/core/engine.py E128 E127 E502 + scrapy/core/scheduler.py + scrapy/core/scraper.py E128 W504 + scrapy/core/spidermw.py E731 E126 + scrapy/core/downloader/__init__.py + scrapy/core/downloader/contextfactory.py E128 E126 + scrapy/core/downloader/middleware.py E502 + scrapy/core/downloader/tls.py E241 + scrapy/core/downloader/webclient.py E731 E128 E126 + scrapy/core/downloader/handlers/ftp.py E128 E127 + scrapy/core/downloader/handlers/s3.py E128 E126 # scrapy/downloadermiddlewares - scrapy/downloadermiddlewares/ajaxcrawl.py E501 - scrapy/downloadermiddlewares/decompression.py E501 - scrapy/downloadermiddlewares/defaultheaders.py E501 - scrapy/downloadermiddlewares/httpcache.py E501 E126 - scrapy/downloadermiddlewares/httpcompression.py E501 E128 - scrapy/downloadermiddlewares/httpproxy.py E501 - scrapy/downloadermiddlewares/redirect.py E501 W504 - scrapy/downloadermiddlewares/retry.py E501 E126 - scrapy/downloadermiddlewares/robotstxt.py E501 - scrapy/downloadermiddlewares/stats.py E501 + scrapy/downloadermiddlewares/httpcache.py E126 + scrapy/downloadermiddlewares/httpcompression.py E128 + scrapy/downloadermiddlewares/redirect.py W504 + scrapy/downloadermiddlewares/retry.py E126 # scrapy/extensions - scrapy/extensions/closespider.py E501 E128 E123 - scrapy/extensions/corestats.py E501 - scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 - scrapy/extensions/memdebug.py E501 - scrapy/extensions/spiderstate.py E501 - scrapy/extensions/telnet.py E501 W504 - scrapy/extensions/throttle.py E501 + scrapy/extensions/closespider.py E128 E123 + scrapy/extensions/feedexport.py E128 + scrapy/extensions/httpcache.py E128 + scrapy/extensions/telnet.py W504 # scrapy/http - scrapy/http/common.py E501 - scrapy/http/cookies.py E501 - scrapy/http/request/__init__.py E501 - scrapy/http/request/form.py E501 E123 - scrapy/http/request/json_request.py E501 - scrapy/http/response/__init__.py E501 E128 - scrapy/http/response/text.py E501 E128 E124 + scrapy/http/request/form.py E123 + scrapy/http/response/__init__.py E128 + scrapy/http/response/text.py E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E731 E501 E402 W504 - scrapy/linkextractors/lxmlhtml.py E501 E731 + scrapy/linkextractors/__init__.py E731 E402 W504 + scrapy/linkextractors/lxmlhtml.py E731 # scrapy/loader - scrapy/loader/__init__.py E501 E128 - scrapy/loader/processors.py E501 + scrapy/loader/__init__.py E128 + scrapy/loader/processors.py # scrapy/pipelines - scrapy/pipelines/__init__.py E501 - scrapy/pipelines/files.py E116 E501 E266 - scrapy/pipelines/images.py E265 E501 - scrapy/pipelines/media.py E125 E501 E266 + scrapy/pipelines/files.py E116 E266 + scrapy/pipelines/images.py E265 + scrapy/pipelines/media.py E125 E266 # scrapy/selector scrapy/selector/__init__.py F403 - scrapy/selector/unified.py E501 E111 + scrapy/selector/unified.py E111 # scrapy/settings - scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 E116 - scrapy/settings/deprecated.py E501 + scrapy/settings/default_settings.py E114 E116 # scrapy/spidermiddlewares - scrapy/spidermiddlewares/httperror.py E501 - scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 W504 - scrapy/spidermiddlewares/urllength.py E501 + scrapy/spidermiddlewares/referer.py E129 W504 # scrapy/spiders - scrapy/spiders/__init__.py E501 E402 - scrapy/spiders/crawl.py E501 - scrapy/spiders/feed.py E501 - scrapy/spiders/sitemap.py E501 + scrapy/spiders/__init__.py E402 # scrapy/utils - scrapy/utils/asyncio.py E501 - scrapy/utils/benchserver.py E501 - scrapy/utils/conf.py E402 E501 - scrapy/utils/datatypes.py E501 - scrapy/utils/decorators.py E501 - scrapy/utils/defer.py E501 E128 - scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/gz.py E501 W504 + scrapy/utils/defer.py E128 + scrapy/utils/deprecate.py E128 E127 E502 + scrapy/utils/gz.py W504 scrapy/utils/http.py F403 - scrapy/utils/httpobj.py E501 - scrapy/utils/iterators.py E501 - scrapy/utils/log.py E128 E501 + scrapy/utils/log.py E128 scrapy/utils/markup.py F403 - scrapy/utils/misc.py E501 scrapy/utils/multipart.py F403 - scrapy/utils/project.py E501 - scrapy/utils/python.py E501 - scrapy/utils/reactor.py E501 - scrapy/utils/reqser.py E501 - scrapy/utils/request.py E127 E501 - scrapy/utils/response.py E501 E128 - scrapy/utils/signal.py E501 E128 - scrapy/utils/sitemap.py E501 - scrapy/utils/spider.py E501 - scrapy/utils/ssl.py E501 - scrapy/utils/test.py E501 - scrapy/utils/url.py E501 F403 E128 F405 + scrapy/utils/request.py E127 + scrapy/utils/response.py E128 + scrapy/utils/signal.py E128 + scrapy/utils/url.py F403 E128 F405 # scrapy - scrapy/__init__.py E402 E501 - scrapy/cmdline.py E501 - scrapy/crawler.py E501 - scrapy/dupefilters.py E501 E202 - scrapy/exceptions.py E501 - scrapy/exporters.py E501 - scrapy/interfaces.py E501 - scrapy/item.py E501 E128 - scrapy/link.py E501 - scrapy/logformatter.py E501 - scrapy/mail.py E402 E128 E501 E502 - scrapy/middleware.py E128 E501 - scrapy/pqueues.py E501 - scrapy/resolver.py E501 - scrapy/responsetypes.py E128 E501 - scrapy/robotstxt.py E501 - scrapy/shell.py E501 - scrapy/signalmanager.py E501 - scrapy/spiderloader.py F841 E501 E126 + scrapy/__init__.py E402 + scrapy/dupefilters.py E202 + scrapy/item.py E128 + scrapy/mail.py E402 E128 E502 + scrapy/middleware.py E128 + scrapy/responsetypes.py E128 + scrapy/spiderloader.py F841 E126 scrapy/squeues.py E128 - scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 tests/mockserver.py E401 E501 E126 E123 diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index a4ec7c8ae..b189e016b 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -165,6 +165,7 @@ if __name__ == '__main__': try: execute() finally: - # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() - # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies + # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit: + # http://doc.pypy.org/en/latest/cpython_differences.html + # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies garbage_collect() diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 9c0014206..83c7b1f19 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods['process_request']: response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): - raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \ - (method.__self__.__class__.__name__, response.__class__.__name__)) + raise _InvalidOutput( + "Middleware %s.process_request must return None, Response or Request, got %s" + % (method.__self__.__class__.__name__, response.__class__.__name__) + ) if response: defer.returnValue(response) defer.returnValue((yield download_func(request=request, spider=spider))) @@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods['process_response']: response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): - raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ - (method.__self__.__class__.__name__, type(response))) + raise _InvalidOutput( + "Middleware %s.process_response must return Response or Request, got %s" + % (method.__self__.__class__.__name__, type(response)) + ) if isinstance(response, Request): defer.returnValue(response) defer.returnValue(response) @@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods['process_exception']: response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): - raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (method.__self__.__class__.__name__, type(response))) + raise _InvalidOutput( + "Middleware %s.process_exception must return None, Response or Request, got %s" + % (method.__self__.__class__.__name__, type(response)) + ) if response: defer.returnValue(response) defer.returnValue(_failure) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0a9af3a62..52cf09844 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. - logger.warning("Failure while parsing robots.txt. " - "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.", - exc_info=sys.exc_info(), - extra={'spider': spider}) + logger.warning( + "Failure while parsing robots.txt. File either contains garbage or " + "is in an encoding other than UTF-8, treating it as an empty file.", + exc_info=sys.exc_info(), + extra={'spider': spider}, + ) robotstxt_body = '' return robotstxt_body diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d76a96451..3d62224bf 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -54,8 +54,12 @@ class Rule: self.process_request = _get_method(self.process_request, spider) self.process_request_argcount = len(get_func_args(self.process_request)) if self.process_request_argcount == 1: - msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' - warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn( + "Rule.process_request should accept two arguments" + " (request, response), accepting only one is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) def _process_request(self, request, response): """ From 182394bcecf556854291f1a0d2e0d2c406bc7b48 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 22 Mar 2020 23:29:30 -0300 Subject: [PATCH 02/20] E501 compliance (tests) --- pytest.ini | 129 ++++++++---------- tests/test_command_shell.py | 4 +- tests/test_crawler.py | 4 +- tests/test_downloader_handlers.py | 10 +- ...st_downloadermiddleware_httpcompression.py | 9 +- tests/test_downloadermiddleware_redirect.py | 10 +- tests/test_exporters.py | 25 +++- tests/test_feedexport.py | 31 +++-- tests/test_http_request.py | 5 +- tests/test_http_response.py | 33 +++-- tests/test_linkextractors.py | 49 ++++--- tests/test_pipeline_files.py | 49 ++++--- tests/test_pipeline_images.py | 39 +++--- tests/test_request_cb_kwargs.py | 10 +- tests/test_responsetypes.py | 9 +- tests/test_selector.py | 32 +++-- tests/test_spider.py | 4 +- tests/test_spidermiddleware_output_chain.py | 20 ++- tests/test_spidermiddleware_referer.py | 86 ++++++++---- tests/test_utils_iterators.py | 25 +++- tests/test_utils_request.py | 8 +- tests/test_utils_sitemap.py | 36 +++-- tests/test_utils_url.py | 23 +++- 23 files changed, 415 insertions(+), 235 deletions(-) diff --git a/pytest.ini b/pytest.ini index 781a45541..6f7e09478 100644 --- a/pytest.ini +++ b/pytest.ini @@ -108,84 +108,63 @@ flake8-ignore = scrapy/spiderloader.py F841 E126 scrapy/squeues.py E128 # tests - tests/__init__.py E402 E501 - tests/mockserver.py E401 E501 E126 E123 + tests/__init__.py E402 + tests/mockserver.py E401 E126 E123 tests/pipelines.py F841 - tests/spiders.py E501 E127 - tests/test_closespider.py E501 E127 - tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 - tests/test_command_shell.py E501 E128 - tests/test_commands.py E128 E501 - tests/test_contracts.py E501 E128 - tests/test_crawl.py E501 E741 E265 - tests/test_crawler.py F841 E501 - tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123 - tests/test_downloadermiddleware.py E501 - tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126 + tests/spiders.py E127 + tests/test_closespider.py E127 + tests/test_command_parse.py E128 + tests/test_command_shell.py E128 + tests/test_commands.py E128 + tests/test_contracts.py E128 + tests/test_crawl.py E741 E265 + tests/test_crawler.py F841 + tests/test_dependencies.py F841 + tests/test_downloader_handlers.py E124 E127 E128 E265 E126 E123 + tests/test_downloadermiddleware_cookies.py E731 E741 E128 E265 E126 tests/test_downloadermiddleware_decompression.py E127 - tests/test_downloadermiddleware_defaultheaders.py E501 - tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 E126 E123 - tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E128 E127 - tests/test_downloadermiddleware_retry.py E501 E128 E126 - tests/test_downloadermiddleware_robotstxt.py E501 - tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 E124 - tests/test_engine.py E401 E501 E128 - tests/test_exporters.py E501 E731 E128 E124 + tests/test_downloadermiddleware_httpcompression.py E126 E123 + tests/test_downloadermiddleware_httpproxy.py E128 + tests/test_downloadermiddleware_redirect.py E128 E127 + tests/test_downloadermiddleware_retry.py E128 E126 + tests/test_dupefilters.py E741 E128 E124 + tests/test_engine.py E401 E128 + tests/test_exporters.py E731 E128 E124 tests/test_extension_telnet.py F841 - tests/test_feedexport.py E501 F841 E241 - tests/test_http_cookies.py E501 - tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 - tests/test_http_response.py E501 E128 E265 + tests/test_feedexport.py F841 E241 + tests/test_http_request.py E402 E127 E128 E128 E126 E123 + tests/test_http_response.py E128 E265 tests/test_item.py E128 F841 - tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E501 E731 E741 E128 E117 E241 - tests/test_logformatter.py E128 E501 E122 - tests/test_mail.py E128 E501 - tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 E126 - tests/test_pipeline_files.py E501 - tests/test_pipeline_images.py F841 E501 - tests/test_pipeline_media.py E501 E741 E731 E128 E502 - tests/test_proxy_connect.py E501 E741 - tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 - tests/test_robotstxt_interface.py E501 E501 - tests/test_scheduler.py E501 E126 E123 - tests/test_selector.py E501 E127 - tests/test_spider.py E501 - tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 - tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 - tests/test_squeues.py E501 E741 - tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 E128 - tests/test_utils_curl.py E501 - tests/test_utils_datatypes.py E402 E501 - tests/test_utils_defer.py E501 F841 - tests/test_utils_deprecate.py F841 E501 - tests/test_utils_http.py E501 E128 W504 - tests/test_utils_iterators.py E501 E128 E129 E241 + tests/test_linkextractors.py E128 E124 + tests/test_loader.py E731 E741 E128 E117 E241 + tests/test_logformatter.py E128 E122 + tests/test_mail.py E128 + tests/test_middleware.py E128 + tests/test_pipeline_crawl.py E128 E126 + tests/test_pipeline_images.py F841 + tests/test_pipeline_media.py E741 E731 E128 E502 + tests/test_proxy_connect.py E741 + tests/test_scheduler.py E126 E123 + tests/test_selector.py E127 + tests/test_spidermiddleware_httperror.py E128 E127 E121 + tests/test_spidermiddleware_offsite.py E128 E111 + tests/test_spidermiddleware_referer.py F841 E125 E201 E124 E241 E121 + tests/test_squeues.py E741 + tests/test_utils_conf.py E128 + tests/test_utils_datatypes.py E402 + tests/test_utils_defer.py F841 + tests/test_utils_deprecate.py F841 + tests/test_utils_http.py E128 W504 + tests/test_utils_iterators.py E128 E129 E241 tests/test_utils_log.py E741 - tests/test_utils_python.py E501 E731 - tests/test_utils_reqser.py E501 E128 - tests/test_utils_request.py E501 E128 - tests/test_utils_response.py E501 + tests/test_utils_python.py E731 + tests/test_utils_reqser.py E128 + tests/test_utils_request.py E128 tests/test_utils_signal.py E741 F841 E731 - tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123 - tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126 - tests/test_cmdline/__init__.py E501 - tests/test_settings/__init__.py E501 E128 - tests/test_spiderloader/__init__.py E128 E501 - tests/test_utils_misc/__init__.py E501 + tests/test_utils_sitemap.py E128 E124 + tests/test_utils_url.py E127 E125 E241 E126 E123 + tests/test_webclient.py E128 E122 E402 E241 E123 E126 + tests/test_cmdline/__init__.py + tests/test_settings/__init__.py E128 + tests/test_spiderloader/__init__.py E128 + tests/test_utils_misc/__init__.py diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..5dc0ba3d2 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -56,7 +56,9 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_redirect_not_follow_302(self): - _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + _, out, _ = yield self.execute( + ['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'] + ) assert out.strip().endswith(b'302') @defer.inlineCallbacks diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 169e763f0..bfc9a609a 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -303,8 +303,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log) self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 29d06bab4..1a7be5c1b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -493,7 +493,10 @@ class Http11TestCase(HttpTestCase): class Https11TestCase(Http11TestCase): scheme = 'https' - tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"' + tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) @defer.inlineCallbacks def test_tls_logging(self): @@ -542,7 +545,10 @@ class Https11InvalidDNSPattern(Https11TestCase): from service_identity.exceptions import CertificateError # noqa: F401 except ImportError: raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) super(Https11InvalidDNSPattern, self).setUp() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 106ca3360..4d43e164f 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -124,7 +124,8 @@ class HttpCompressionTest(TestCase): 'Content-Encoding': 'gzip', } f = BytesIO() - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) zf.close() @@ -142,7 +143,8 @@ class HttpCompressionTest(TestCase): 'Content-Encoding': 'gzip', } f = BytesIO() - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) zf.close() @@ -158,7 +160,8 @@ class HttpCompressionTest(TestCase): headers = { 'Content-Encoding': 'identity', } - plainbody = b"""Some page""" + plainbody = (b'Some page' + b'') respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody) response = respcls("http://www.example.com/index", headers=headers, body=plainbody) request = Request("http://www.example.com/index") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 053e26fc3..3044c5b21 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -151,7 +151,10 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, 'http://scrapytest.org/redirected') self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') - self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + self.assertEqual( + req3.meta['redirect_urls'], + ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + ) def test_redirect_reasons(self): req1 = Request('http://scrapytest.org/first') @@ -282,7 +285,10 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, 'http://scrapytest.org/redirected') self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') - self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + self.assertEqual( + req3.meta['redirect_urls'], + ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + ) def test_redirect_reasons(self): req1 = Request('http://scrapytest.org/first') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6e2507508..b8301a567 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -111,7 +111,10 @@ class PythonItemExporterTest(BaseItemExporterTest): ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) - self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'} + ) self.assertEqual(type(exported['age']), dict) self.assertEqual(type(exported['age']['age']), dict) @@ -121,7 +124,10 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) @@ -131,7 +137,10 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual( + exported, + {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) @@ -327,13 +336,19 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = b'\n22John\xc2\xa3' + expected_value = ( + b'\n' + b'22John\xc2\xa3' + ) self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - b'\nJohn\xc2\xa3Doe' + ( + b'\n' + b'John\xc2\xa3Doe' + ) ) def test_nested_item(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c5589e52f..e248969db 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -718,10 +718,13 @@ class FeedExportTest(unittest.TestCase): header = ['foo'] formats = { - 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), - 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'\nTest\xd6'.encode('utf-8'), - 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), + 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), + 'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'), + 'xml': ( + '\n' + 'Test\xd6' + ).encode('utf-8'), + 'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'), } for fmt, expected in formats.items(): @@ -735,10 +738,13 @@ class FeedExportTest(unittest.TestCase): self.assertEqual(expected, data[fmt]) formats = { - 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), - 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'\nTest\xd6'.encode('latin-1'), - 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), + 'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'), + 'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'), + 'xml': ( + '\n' + 'Test\xd6' + ).encode('latin-1'), + 'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'), } for fmt, expected in formats.items(): @@ -757,9 +763,12 @@ class FeedExportTest(unittest.TestCase): items = [dict({'foo': u'FOO', 'bar': u'BAR'})] formats = { - 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'\n\n \n FOO\n \n'.encode('latin-1'), - 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), + 'xml': ( + '\n' + '\n \n FOO\n \n' + ).encode('latin-1'), + 'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } settings = { diff --git a/tests/test_http_request.py b/tests/test_http_request.py index cc2cddda4..87d98d6ee 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1262,7 +1262,10 @@ class XmlRpcRequestTest(RequestTest): class JsonRequestTest(RequestTest): request_class = JsonRequest default_method = 'GET' - default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} + default_headers = { + b'Content-Type': [b'application/json'], + b'Accept': [b'application/json, text/javascript, */*; q=0.01'], + } def setUp(self): warnings.simplefilter("always") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index eafc3560e..a6b717bd1 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -25,7 +25,11 @@ class BaseResponseTest(unittest.TestCase): self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) # test presence of all optional parameters - self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class)) + self.assertTrue( + isinstance( + self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class + ) + ) r = self.response_class("http://www.example.com") assert isinstance(r.url, str) @@ -323,13 +327,16 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) + resp = self.response_class(u"http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}) self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) + resp = self.response_class(u"http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') def test_unicode_body(self): - unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442' + unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' + '\u0442\u0435\u043a\u0441\u0442') self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body') original_string = unicode_string.encode('cp1251') @@ -344,13 +351,18 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r1.text, unicode_string) def test_encoding(self): - r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3") + r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}) r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") - r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=b"\xa3") + r3 = self.response_class("http://www.example.com", body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body=b"\xc2\xa3") - r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body=b"\xa8D") - r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body=b"\xa8D") + r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}) + r6 = self.response_class("http://www.example.com", body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}) + r7 = self.response_class("http://www.example.com", body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -685,7 +697,8 @@ class HtmlResponseTest(TextResponseTest): body = b"""Some page Price: \xa3100' """ - r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body) + r3 = self.response_class("http://www.example.com", body=body, + headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self._assert_response_values(r3, 'iso-8859-1', body) # make sure replace() preserves the encoding of the original response diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 53968e60e..ed5f2560a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -279,8 +279,8 @@ class Base: def test_process_value(self): """Test restrict_xpaths with encodings""" html = b""" - Link text - About us +Text +About us """ response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') @@ -291,7 +291,7 @@ class Base: lx = self.extractor_cls(process_value=process_value) self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/other/page.html', text='Link text')]) + [Link(url='http://example.org/other/page.html', text='Text')]) def test_base_url_with_restrict_xpaths(self): html = b"""Page title<title><base href="http://otherdomain.com/base/" /> @@ -332,7 +332,10 @@ class Base: self.assertEqual(lx.extract_links(self.response), []) def test_tags(self): - html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>""" + html = ( + b'<html><area href="sample1.html"></area>' + b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>' + ) response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags=None) @@ -413,24 +416,34 @@ class Base: response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', + nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', + nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) response = XmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', + nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', + nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) def test_link_wrong_href(self): html = b""" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index f155db4ce..34fc8dcb6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -38,27 +38,36 @@ class FilesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') - self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")), - 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') - self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")), - 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), - 'full/76c00cef2ef669ae65052661f68d451162829507') - self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ + self.assertEqual( + file_path(Request("https://dev.mydeco.com/mydeco.pdf")), + 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + self.assertEqual( + file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")), + 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")), + 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object()), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), + 'full/76c00cef2ef669ae65052661f68d451162829507') + self.assertEqual( + file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")), - 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') + 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 5018d6802..e1c3a55dd 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -41,22 +41,29 @@ class ImagesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual(file_path(Request("https://dev.mydeco.com/mydeco.gif")), - 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') - self.assertEqual(file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")), - 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg') - self.assertEqual(file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")), - 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg') - self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') - self.assertEqual(file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/mydeco.gif")), + 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual( + file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")), + 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg') + self.assertEqual( + file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")), + 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg') + self.assertEqual( + file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + self.assertEqual( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object()), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') def test_thumbnail_name(self): thumb_path = self.pipeline.thumb_path diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a3ddd50f4..bd49179aa 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -158,6 +158,12 @@ class CallbackKeywordArgumentsTestCase(TestCase): if key in line.getMessage(): exceptions[key] = line self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) - self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'") + self.assertEqual( + str(exceptions['takes_less'].exc_info[1]), + "parse_takes_less() got an unexpected keyword argument 'number'" + ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'") + self.assertEqual( + str(exceptions['takes_more'].exc_info[1]), + "parse_takes_more() missing 1 required positional argument: 'other'" + ) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 8cdf7a176..9f4d56412 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -64,8 +64,9 @@ class ResponseTypesTest(unittest.TestCase): def test_from_headers(self): mappings = [ ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse), - ({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse), ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response), + ({'Content-Type': ['application/octet-stream'], + 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse), ] for source, cls in mappings: source = Headers(source) @@ -77,8 +78,10 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ ({'url': 'http://www.example.com/data.csv'}, TextResponse), # headers takes precedence over url - ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse), - ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response), + ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), + 'url': 'http://www.example.com/item/'}, HtmlResponse), + ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), + 'url': 'http://www.example.com/page/'}, Response), ] diff --git a/tests/test_selector.py b/tests/test_selector.py index 09c2546fb..3629feb16 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -19,18 +19,26 @@ class SelectorTestCase(unittest.TestCase): for x in xl: assert isinstance(x, Selector) - self.assertEqual(sel.xpath('//input').getall(), - [x.get() for x in sel.xpath('//input')]) - - self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")], - [u'a']) - self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0']) - - self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), - [u'xpathrules']) - self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12']) + self.assertEqual( + sel.xpath('//input').getall(), + [x.get() for x in sel.xpath('//input')] + ) + self.assertEqual( + [x.get() for x in sel.xpath("//input[@name='a']/@name")], + [u'a'] + ) + self.assertEqual( + [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + [u'12.0'] + ) + self.assertEqual( + sel.xpath("concat('xpath', 'rules')").getall(), + [u'xpathrules'] + ) + self.assertEqual( + [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + [u'12'] + ) def test_root_base_url(self): body = b'<html><form action="/path"><input name="a" /></form></html>' diff --git a/tests/test_spider.py b/tests/test_spider.py index bb00c8f42..805d70459 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -120,7 +120,9 @@ class XMLFeedSpiderTest(SpiderTest): body = b"""<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84" xmlns:y="http://www.example.com/schemas/extras/1.0"> - <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated><other value="bar" y:custom="fuu"/></url> + <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated> + <other value="bar" y:custom="fuu"/> + </url> <url><loc>http://www.example.com/</loc><y:updated>2009-08-16</updated><other value="foo"/></url> </urlset>""" response = XmlResponse(url='http://example.com/sitemap.xml', body=body) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index b26353d6c..78d2de347 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -385,9 +385,15 @@ class TestSpiderMiddleware(TestCase): log4 = yield self.crawl_log(GeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 2", str(log4)) self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", + str(log4)) + self.assertNotIn( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught", + str(log4)) + self.assertNotIn( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", + str(log4)) item_from_callback = {'processed': [ 'parse-first-item', 'GeneratorFailMiddleware.process_spider_output', @@ -414,9 +420,13 @@ class TestSpiderMiddleware(TestCase): log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 1", str(log5)) self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5)) - self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertIn( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", + str(log5)) self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5)) - self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertNotIn( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", + str(log5)) item_recovered = {'processed': [ 'NotGeneratorRecoverMiddleware.process_spider_exception', 'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4c6ede70b..3974f7884 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -114,12 +114,16 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: do NOT send referrer ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), @@ -139,8 +143,12 @@ class MixinSameOrigin: ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ( + 'https://user:password@example.com/page.html', + 'https://example.com/not-page.html', + b'https://example.com/page.html', + ), ] @@ -179,12 +187,16 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: send origin as referrer ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), @@ -205,21 +217,33 @@ class MixinOriginWhenCrossOrigin: ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + ( + 'https://user:password@example5.com/page.html', + 'https://example5.com/not-page.html', + b'https://example5.com/page.html', + ), # TLS to non-TLS downgrade: send origin - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ( + 'https://user:password@example5.com/page.html', + 'http://example5.com/not-page.html', + b'https://example5.com/', + ), ] class MixinStrictOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ( + 'http://example.com:8888/page.html', + 'http://example.com:8888/not-page.html', + b'http://example.com:8888/page.html', + ), # Different host: send origin as referrer ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), @@ -248,7 +272,11 @@ class MixinStrictOriginWhenCrossOrigin: ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + ( + 'https://user:password@example5.com/page.html', + 'https://example5.com/not-page.html', + b'https://example5.com/page.html', + ), # TLS to non-TLS downgrade: send nothing ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), @@ -281,8 +309,16 @@ class MixinUnsafeUrl: ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), # test for user/password stripping - ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), - ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ( + 'http://user:password@example4.com/page.html', + 'https://not.example4.com/', + b'http://example4.com/page.html', + ), + ( + 'https://user:password@example4.com/page.html', + 'http://scrapy.org/', + b'https://example4.com/page.html', + ), ] diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 33fc4d570..7c4011142 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -17,7 +17,8 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter(self): body = b"""<?xml version="1.0" encoding="UTF-8"?>\ - <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\ + <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="someschmea.xsd">\ <product id="001">\ <type>Type 1</type>\ <name>Name 1</name>\ @@ -107,7 +108,10 @@ class XmliterTestCase(unittest.TestCase): (u'27', [u'A'], [u'27'])]) def test_xmliter_text(self): - body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" + body = ( + '<?xml version="1.0" encoding="UTF-8"?>' + '<products><product>one</product><product>two</product></products>' + ) self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) @@ -139,7 +143,10 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('title/text()').getall(), ['Item 1']) self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1']) self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual( + node.xpath('g:image_link/text()').getall(), + ['http://www.mydummycompany.com/images/item1.jpg'] + ) self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1']) self.assertEqual(node.xpath('g:price/text()').getall(), ['400']) self.assertEqual(node.xpath('image_link/text()').getall(), []) @@ -147,7 +154,10 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('price/text()').getall(), []) def test_xmliter_exception(self): - body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" + body = ( + '<?xml version="1.0" encoding="UTF-8"?>' + '<products><product>one</product><product>two</product></products>' + ) iter = self.xmliter(body, 'product') next(iter) @@ -160,7 +170,12 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(AssertionError, next, i) def test_xmliter_encoding(self): - body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n' + body = ( + b'<?xml version="1.0" encoding="ISO-8859-9"?>\n' + b'<xml>\n' + b' <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n' + b'</xml>\n\n' + ) response = XmlResponse('http://www.example.com', body=body) self.assertEqual( next(self.xmliter(response, 'item')).get(), diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 45f0f59e4..3b1994019 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -75,8 +75,12 @@ class UtilsRequestTest(unittest.TestCase): r1 = Request("http://www.example.com/some/page.html?arg=1") self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n') - r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body") - self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Request("http://www.example.com", method='POST', + headers={"Content-type": b"text/html"}, body=b"Some body") + self.assertEqual( + request_httprepr(r1), + b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body' + ) def test_request_httprepr_for_non_http_request(self): # the representation is not important but it must not fail. diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index db323ab31..4c09d33b2 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -22,8 +22,14 @@ class SitemapTest(unittest.TestCase): </url> </urlset>""") assert s.type == 'urlset' - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', + 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, + ] + ) def test_sitemap_index(self): s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?> @@ -38,7 +44,13 @@ class SitemapTest(unittest.TestCase): </sitemap> </sitemapindex>""") assert s.type == 'sitemapindex' - self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}]) + self.assertEqual( + list(s), + [ + {'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, + {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}, + ] + ) def test_sitemap_strip(self): """Assert we can deal with trailing spaces inside <loc> tags - we've @@ -195,11 +207,19 @@ Disallow: /forum/active/ </url> </urlset>""") - self.assertEqual(list(s), [ - {'loc': 'http://www.example.com/english/', - 'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/'] - } - ]) + self.assertEqual( + list(s), + [ + { + 'loc': 'http://www.example.com/english/', + 'alternate': [ + 'http://www.example.com/deutsch/', + 'http://www.example.com/schweiz-deutsch/', + 'http://www.example.com/english/', + ], + } + ] + ) def test_xml_entity_expansion(self): s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?> diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 7abff8281..a12b6dda2 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -28,7 +28,10 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) - url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' + url = ( + 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20' + 'javascript:%20document.orderform_2581_1190810811.submit%28%29' + ) self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com'])) @@ -105,8 +108,10 @@ class AddHttpIfNoScheme(unittest.TestCase): 'http://username:password@www.example.com') def test_complete_url(self): - self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' + ) def test_preserve_http(self): self.assertEqual(add_http_if_no_scheme('http://www.example.com'), @@ -137,8 +142,10 @@ class AddHttpIfNoScheme(unittest.TestCase): 'http://username:password@www.example.com') def test_preserve_http_complete_url(self): - self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' + ) def test_protocol_relative(self): self.assertEqual(add_http_if_no_scheme('//www.example.com'), @@ -169,8 +176,10 @@ class AddHttpIfNoScheme(unittest.TestCase): 'http://username:password@www.example.com') def test_protocol_relative_complete_url(self): - self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' + ) def test_preserve_https(self): self.assertEqual(add_http_if_no_scheme('https://www.example.com'), From 4663f0b9df7f687bf05ca57e5b6e8442b0997df7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 22 Mar 2020 23:38:51 -0300 Subject: [PATCH 03/20] Update pytest.ini after removing E501 --- pytest.ini | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index 6f7e09478..1b00bfb84 100644 --- a/pytest.ini +++ b/pytest.ini @@ -33,7 +33,6 @@ flake8-ignore = scrapy/commands/fetch.py E401 E128 E731 scrapy/commands/genspider.py E128 E502 scrapy/commands/parse.py E128 E731 - scrapy/commands/runspider.py scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E502 scrapy/commands/startproject.py E127 E128 @@ -43,10 +42,8 @@ flake8-ignore = scrapy/contracts/default.py E128 # scrapy/core scrapy/core/engine.py E128 E127 E502 - scrapy/core/scheduler.py scrapy/core/scraper.py E128 W504 scrapy/core/spidermw.py E731 E126 - scrapy/core/downloader/__init__.py scrapy/core/downloader/contextfactory.py E128 E126 scrapy/core/downloader/middleware.py E502 scrapy/core/downloader/tls.py E241 @@ -72,7 +69,6 @@ flake8-ignore = scrapy/linkextractors/lxmlhtml.py E731 # scrapy/loader scrapy/loader/__init__.py E128 - scrapy/loader/processors.py # scrapy/pipelines scrapy/pipelines/files.py E116 E266 scrapy/pipelines/images.py E265 @@ -87,6 +83,7 @@ flake8-ignore = # scrapy/spiders scrapy/spiders/__init__.py E402 # scrapy/utils + scrapy/utils/conf.py E402 scrapy/utils/defer.py E128 scrapy/utils/deprecate.py E128 E127 E502 scrapy/utils/gz.py W504 @@ -164,7 +161,5 @@ flake8-ignore = tests/test_utils_sitemap.py E128 E124 tests/test_utils_url.py E127 E125 E241 E126 E123 tests/test_webclient.py E128 E122 E402 E241 E123 E126 - tests/test_cmdline/__init__.py tests/test_settings/__init__.py E128 tests/test_spiderloader/__init__.py E128 - tests/test_utils_misc/__init__.py From ee510cf0d6dda87bd5d472d177c04ceef742d1e3 Mon Sep 17 00:00:00 2001 From: elacuesta <elacuesta@users.noreply.github.com> Date: Tue, 24 Mar 2020 13:31:44 -0300 Subject: [PATCH 04/20] Update scrapy/spiders/crawl.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves <adrian@chaves.io> --- scrapy/spiders/crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 3d62224bf..cb021a5a7 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -55,8 +55,8 @@ class Rule: self.process_request_argcount = len(get_func_args(self.process_request)) if self.process_request_argcount == 1: warnings.warn( - "Rule.process_request should accept two arguments" - " (request, response), accepting only one is deprecated", + "Rule.process_request should accept two arguments " + "(request, response), accepting only one is deprecated", category=ScrapyDeprecationWarning, stacklevel=2, ) From 83a0cc6cdf4b8d55ebc594f2635beb75d93898cf Mon Sep 17 00:00:00 2001 From: ilias-ant <ilias.antonopoulos@yahoo.gr> Date: Sun, 12 Apr 2020 23:22:17 +0300 Subject: [PATCH 05/20] Add status to files information --- docs/topics/media-pipeline.rst | 13 +++++++++++-- scrapy/pipelines/files.py | 4 ++-- tests/test_pipeline_crawl.py | 5 +++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index cd84905c5..86550d7a4 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -50,7 +50,7 @@ this: 4. When the files are downloaded, another field (``files``) will be populated with the results. This field will contain a list of dicts with information about the downloaded files, such as the downloaded path, the original - scraped url (taken from the ``file_urls`` field) , and the file checksum. + scraped url (taken from the ``file_urls`` field), the file checksum and the file status. The files in the list of the ``files`` field will retain the same order of the original ``file_urls`` field. If some file failed downloading, an error will be logged and the file won't be present in the ``files`` field. @@ -470,6 +470,14 @@ See here the methods that you can override in your custom Files Pipeline: * ``checksum`` - a `MD5 hash`_ of the image contents + * ``status`` - the file status indication. It can be one of the following: + + * ``downloaded`` - file was downloaded. + * ``uptodate`` - file was not downloaded, as it was downloaded recently, + according to the file expiration policy. + * ``cached`` - file was already scheduled for download, by another item + sharing the same file. + The list of tuples received by :meth:`~item_completed` is guaranteed to retain the same order of the requests returned from the :meth:`~get_media_requests` method. @@ -479,7 +487,8 @@ See here the methods that you can override in your custom Files Pipeline: [(True, {'checksum': '2b00042f7481c7b056c4b410d28f33cf', 'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg', - 'url': 'http://www.example.com/files/product1.pdf'}), + 'url': 'http://www.example.com/files/product1.pdf', + 'status': 'downloaded'}), (False, Failure(...))] diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 101bf5fbc..747dfa065 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -417,7 +417,7 @@ class FilesPipeline(MediaPipeline): self.inc_stats(info.spider, 'uptodate') checksum = result.get('checksum', None) - return {'url': request.url, 'path': path, 'checksum': checksum} + return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} path = self.file_path(request, info=info) dfd = defer.maybeDeferred(self.store.stat_file, path, info) @@ -494,7 +494,7 @@ class FilesPipeline(MediaPipeline): ) raise FileException(str(exc)) - return {'url': request.url, 'path': path, 'checksum': checksum} + return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status} def inc_stats(self, spider, status): spider.crawler.stats.inc_value('file_count', spider=spider) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 962c33144..73b03e9d2 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -94,6 +94,11 @@ class FileDownloadCrawlTestCase(TestCase): file_dl_success = 'File (downloaded): Downloaded file from' self.assertEqual(logs.count(file_dl_success), 3) + # check that the images/files status is `downloaded` + for item in items: + for i in item[self.media_key]: + self.assertEqual(i['status'], 'downloaded') + # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: checksums = set( From f242751b625c4a559d9e80f62a79223f911a8551 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 15 Apr 2020 09:39:28 -0300 Subject: [PATCH 06/20] Remove empty line --- pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 47f34e8f1..a9014ae55 100644 --- a/pytest.ini +++ b/pytest.ini @@ -68,7 +68,6 @@ flake8-ignore = # scrapy/loader scrapy/loader/__init__.py E128 # scrapy/pipelines - scrapy/pipelines/files.py E116 scrapy/pipelines/media.py E125 # scrapy/selector From 773ddf77ba36613a90d35e583b29e9020cc2aace Mon Sep 17 00:00:00 2001 From: ilias-ant <ilias.antonopoulos@yahoo.gr> Date: Sun, 19 Apr 2020 14:14:17 +0300 Subject: [PATCH 07/20] added more tests to cover the (downloaded, uptodate, cached) status functionality --- tests/test_pipeline_files.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index f155db4ce..9eb6462f2 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -84,6 +84,7 @@ class FilesPipelineTestCase(unittest.TestCase): result = yield self.pipeline.process_item(item, None) self.assertEqual(result['files'][0]['checksum'], 'abc') + self.assertEqual(result['files'][0]['status'], 'uptodate') for p in patchers: p.stop() @@ -105,6 +106,29 @@ class FilesPipelineTestCase(unittest.TestCase): result = yield self.pipeline.process_item(item, None) self.assertNotEqual(result['files'][0]['checksum'], 'abc') + self.assertEqual(result['files'][0]['status'], 'downloaded') + + for p in patchers: + p.stop() + + @defer.inlineCallbacks + def test_file_cached(self): + item_url = "http://example.com/file3.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), + mock.patch.object(FSFilesStore, 'stat_file', return_value={ + 'checksum': 'abc', + 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url, flags=['cached'])]) + ] + for p in patchers: + p.start() + + result = yield self.pipeline.process_item(item, None) + self.assertNotEqual(result['files'][0]['checksum'], 'abc') + self.assertEqual(result['files'][0]['status'], 'cached') for p in patchers: p.stop() @@ -403,10 +427,10 @@ def _create_item_with_files(*files): return item -def _prepare_request_object(item_url): +def _prepare_request_object(item_url, flags=None): return Request( item_url, - meta={'response': Response(item_url, status=200, body=b'data')}) + meta={'response': Response(item_url, status=200, body=b'data', flags=flags)}) if __name__ == "__main__": From f75941f79dfeb931649f9e8880cdbd1b0a8e4681 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 30 Apr 2020 11:56:52 -0300 Subject: [PATCH 08/20] Deprecate scrapy.item.BaseItem --- scrapy/item.py | 7 ++++++- tests/test_item.py | 25 ++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index 748368932..3558b2231 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -28,7 +28,12 @@ class BaseItem(object_ref): Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. """ - pass + + def __new__(cls, *args, **kwargs): + if issubclass(cls, BaseItem) and not (issubclass(cls, Item) or issubclass(cls, DictItem)): + warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', + ScrapyDeprecationWarning, stacklevel=2) + return super(BaseItem, cls).__new__(cls, *args, **kwargs) class Field(dict): diff --git a/tests/test_item.py b/tests/test_item.py index 4017f6e84..1220bc98a 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -4,7 +4,7 @@ from unittest import mock from warnings import catch_warnings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, BaseItem, DictItem, Field, Item, ItemMeta PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) @@ -131,12 +131,12 @@ class ItemTest(unittest.TestCase): self.assertSortedEqual(list(item.values()), [u'New']) def test_metaclass_inheritance(self): - class BaseItem(Item): + class ParentItem(Item): name = Field() keys = Field() values = Field() - class TestItem(BaseItem): + class TestItem(ParentItem): keys = Field() i = TestItem() @@ -321,13 +321,28 @@ class DictItemTest(unittest.TestCase): def test_deprecation_warning(self): with catch_warnings(record=True) as warnings: - dict_item = DictItem() + DictItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) with catch_warnings(record=True) as warnings: class SubclassedDictItem(DictItem): pass - subclassed_dict_item = SubclassedDictItem() + SubclassedDictItem() + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + + +class BaseItemTest(unittest.TestCase): + + def test_deprecation_warning(self): + with catch_warnings(record=True) as warnings: + BaseItem() + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + class SubclassedBaseItem(BaseItem): + pass + SubclassedBaseItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) From 622ce860669b8bd7fc581b74d414aef24f4fb041 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Mon, 4 May 2020 16:22:24 -0300 Subject: [PATCH 09/20] Test: make sure scrapy.item.Item does not issue a deprecation warning --- tests/test_item.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_item.py b/tests/test_item.py index 1220bc98a..f35a2b9f9 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -347,5 +347,18 @@ class BaseItemTest(unittest.TestCase): self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) +class ItemNoDeprecationWarningTest(unittest.TestCase): + + def test_no_deprecation_warning(self): + with catch_warnings(record=True) as warnings: + Item() + self.assertEqual(len(warnings), 0) + with catch_warnings(record=True) as warnings: + class SubclassedItem(Item): + pass + SubclassedItem() + self.assertEqual(len(warnings), 0) + + if __name__ == "__main__": unittest.main() From 7988c676a956457843416f8586544a58bf121f63 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Tue, 5 May 2020 13:11:01 -0300 Subject: [PATCH 10/20] Update Item docstring, update BaseItem occurrences --- docs/faq.rst | 4 ++-- pytest.ini | 2 +- scrapy/item.py | 35 ++++++++++++++++++----------------- scrapy/spiders/feed.py | 2 +- tests/test_loader.py | 2 +- tests/test_utils_spider.py | 4 ++-- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 75a0f4864..79ef6ca85 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -342,14 +342,14 @@ method for this purpose. For example:: from copy import deepcopy - from scrapy.item import BaseItem + from scrapy.item import Item class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: - if isinstance(item, (BaseItem, dict)): + if isinstance(item, (Item, dict)): for _ in range(item['multiply_by']): yield deepcopy(item) diff --git a/pytest.ini b/pytest.ini index e8911ee3f..5a86ce2a7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -153,7 +153,7 @@ flake8-ignore = scrapy/exceptions.py E501 scrapy/exporters.py E501 scrapy/interfaces.py E501 - scrapy/item.py E501 E128 + scrapy/item.py E501 scrapy/link.py E501 scrapy/logformatter.py E501 scrapy/mail.py E402 E128 E501 diff --git a/scrapy/item.py b/scrapy/item.py index 3558b2231..46d20d017 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -15,18 +15,8 @@ from scrapy.utils.trackref import object_ref class BaseItem(object_ref): - """Base class for all scraped items. - - In Scrapy, an object is considered an *item* if it is an instance of either - :class:`BaseItem` or :class:`dict`. For example, when the output of a - spider callback is evaluated, only instances of :class:`BaseItem` or - :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`BaseItem` or :class:`dict`. - - Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be - :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. + """ + Deprecated, please use :class:`scrapy.item.Item` instead """ def __new__(cls, *args, **kwargs): @@ -91,8 +81,7 @@ class DictItem(MutableMapping, BaseItem): if key in self.fields: self._values[key] = value else: - raise KeyError("%s does not support field: %s" % - (self.__class__.__name__, key)) + raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) def __delitem__(self, key): del self._values[key] @@ -104,8 +93,7 @@ class DictItem(MutableMapping, BaseItem): def __setattr__(self, name, value): if not name.startswith('_'): - raise AttributeError("Use item[%r] = %r to set field value" % - (name, value)) + raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) super(DictItem, self).__setattr__(name, value) def __len__(self): @@ -132,4 +120,17 @@ class DictItem(MutableMapping, BaseItem): class Item(DictItem, metaclass=ItemMeta): - pass + """ + Base class for scraped items. + + In Scrapy, an object is considered an *item* if it is an instance of either + :class:`Item` or :class:`dict`. For example, when the output of a + spider callback is evaluated, only instances of :class:`Item` or + :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`. + + If you need instances of a custom class to be considered items by Scrapy, + you must inherit from either :class:`Item` or :class:`dict`. + + Unlike instances of :class:`dict`, instances of :class:`Item` may be + :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. + """ diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index c566f0236..a4ff8010d 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -52,7 +52,7 @@ class XMLFeedSpider(Spider): """This method is called for the nodes matching the provided tag name (itertag). Receives the response and an Selector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either a BaseItem, a Request, or a list + This method must return either an item, a request, or a list containing any of them. """ diff --git a/tests/test_loader.py b/tests/test_loader.py index 701d568dc..f14714c75 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -601,7 +601,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader): class NoInputReprocessingFromItemTest(unittest.TestCase): """ - Loaders initialized from loaded items must not reprocess fields (BaseItem instances) + Loaders initialized from loaded items must not reprocess fields (Item instances) """ def test_avoid_reprocessing_with_initial_values_single(self): il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo')) diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index ee7d17062..3c87268ab 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -2,7 +2,7 @@ import unittest from scrapy import Spider from scrapy.http import Request -from scrapy.item import BaseItem +from scrapy.item import Item from scrapy.utils.spider import iterate_spider_output, iter_spider_classes @@ -17,7 +17,7 @@ class MySpider2(Spider): class UtilsSpidersTestCase(unittest.TestCase): def test_iterate_spider_output(self): - i = BaseItem() + i = Item() r = Request('http://scrapytest.org') o = object() From 8d1e3ee0dd8d48428f719a0fffda964d30ab956d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 6 May 2020 09:24:32 -0300 Subject: [PATCH 11/20] Remove deprecated BaseItem from the docs --- docs/topics/items.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 78612f524..0941a8a1b 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -257,6 +257,4 @@ Field objects Other classes related to Item ============================= -.. autoclass:: BaseItem - .. autoclass:: ItemMeta From 5256eae60d3685de51c1f3891abe157e15d14def Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 7 May 2020 14:37:41 -0300 Subject: [PATCH 12/20] Meta class to handle isinstance checks for BaseItem --- pytest.ini | 2 +- scrapy/commands/parse.py | 4 +-- scrapy/contracts/default.py | 8 +++--- scrapy/core/scraper.py | 4 +-- scrapy/exporters.py | 4 +-- scrapy/item.py | 20 +++++++++++-- scrapy/shell.py | 5 ++-- scrapy/utils/misc.py | 4 +-- scrapy/utils/serialize.py | 4 +-- tests/test_item.py | 56 +++++++++++++++++++++++++++++++++---- 10 files changed, 85 insertions(+), 26 deletions(-) diff --git a/pytest.ini b/pytest.ini index 5a86ce2a7..292dbce41 100644 --- a/pytest.ini +++ b/pytest.ini @@ -204,7 +204,7 @@ flake8-ignore = tests/test_http_headers.py E501 tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 tests/test_http_response.py E501 E128 - tests/test_item.py E128 F841 + tests/test_item.py E128 F841 E501 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 tests/test_loader.py E501 E741 E128 E117 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 1cefed106..098827ab9 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,7 +5,7 @@ from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request -from scrapy.item import BaseItem +from scrapy.item import _BaseItem from scrapy.utils import display from scrapy.utils.conf import arglist_to_dict from scrapy.utils.spider import iterate_spider_output, spidercls_for_request @@ -117,7 +117,7 @@ class Command(ScrapyCommand): items, requests = [], [] for x in iterate_spider_output(callback(response, **cb_kwargs)): - if isinstance(x, (BaseItem, dict)): + if isinstance(x, (_BaseItem, dict)): items.append(x) elif isinstance(x, Request): requests.append(x) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index a1b0f8f22..cdc2bac15 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,6 +1,6 @@ import json -from scrapy.item import BaseItem +from scrapy.item import _BaseItem from scrapy.http import Request from scrapy.exceptions import ContractFail @@ -51,8 +51,8 @@ class ReturnsContract(Contract): objects = { 'request': Request, 'requests': Request, - 'item': (BaseItem, dict), - 'items': (BaseItem, dict), + 'item': (_BaseItem, dict), + 'items': (_BaseItem, dict), } def __init__(self, *args, **kwargs): @@ -103,7 +103,7 @@ class ScrapesContract(Contract): def post_process(self, output): for x in output: - if isinstance(x, (BaseItem, dict)): + if isinstance(x, (_BaseItem, dict)): missing = [arg for arg in self.args if arg not in x] if missing: raise ContractFail( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index edbb4dd66..6785e103d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -14,7 +14,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy import signals from scrapy.http import Request, Response -from scrapy.item import BaseItem +from scrapy.item import _BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager @@ -191,7 +191,7 @@ class Scraper: """ if isinstance(output, Request): self.crawler.engine.crawl(request=output, spider=spider) - elif isinstance(output, (BaseItem, dict)): + elif isinstance(output, (_BaseItem, dict)): self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) dfd.addBoth(self._itemproc_finished, output, response, spider) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0cb6cef98..4731b925a 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -12,7 +12,7 @@ from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, is_listlike -from scrapy.item import BaseItem +from scrapy.item import _BaseItem from scrapy.exceptions import ScrapyDeprecationWarning @@ -312,7 +312,7 @@ class PythonItemExporter(BaseItemExporter): return serializer(value) def _serialize_value(self, value): - if isinstance(value, BaseItem): + if isinstance(value, _BaseItem): return self.export_item(value) if isinstance(value, dict): return dict(self._serialize_dict(value)) diff --git a/scrapy/item.py b/scrapy/item.py index 46d20d017..f468ff86f 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -14,7 +14,23 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -class BaseItem(object_ref): +class _BaseItem(object_ref): + """ + Temporary class used internally to avoid the deprecation + warning raised by isinstance checks using BaseItem. + """ + pass + + +class _BaseItemMeta(ABCMeta): + def __instancecheck__(cls, instance): + if cls is BaseItem: + warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', + ScrapyDeprecationWarning, stacklevel=2) + return super().__instancecheck__(instance) + + +class BaseItem(_BaseItem, metaclass=_BaseItemMeta): """ Deprecated, please use :class:`scrapy.item.Item` instead """ @@ -30,7 +46,7 @@ class Field(dict): """Container of field metadata""" -class ItemMeta(ABCMeta): +class ItemMeta(_BaseItemMeta): """Metaclass_ of :class:`Item` that handles field definitions. .. _metaclass: https://realpython.com/python-metaclasses diff --git a/scrapy/shell.py b/scrapy/shell.py index 08ce89481..83afb74c9 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -13,7 +13,7 @@ from w3lib.url import any_to_uri from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response -from scrapy.item import BaseItem +from scrapy.item import _BaseItem from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.console import start_python_console @@ -26,8 +26,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS class Shell: - relevant_classes = (Crawler, Spider, Request, Response, BaseItem, - Settings) + relevant_classes = (Crawler, Spider, Request, Response, _BaseItem, Settings) def __init__(self, crawler, update_vars=None, code=None): self.crawler = crawler diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 52cfba208..bfe3ccd40 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -14,10 +14,10 @@ from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode -from scrapy.item import BaseItem +from scrapy.item import _BaseItem -_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes +_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes def arg_to_iter(arg): diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 9dd72ea71..bf73dfa18 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -5,7 +5,7 @@ import decimal from twisted.internet import defer from scrapy.http import Request, Response -from scrapy.item import BaseItem +from scrapy.item import _BaseItem class ScrapyJSONEncoder(json.JSONEncoder): @@ -26,7 +26,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): return str(o) elif isinstance(o, defer.Deferred): return str(o) - elif isinstance(o, BaseItem): + elif isinstance(o, _BaseItem): return dict(o) elif isinstance(o, Request): return "<%s %s %s>" % (type(o).__name__, o.method, o.url) diff --git a/tests/test_item.py b/tests/test_item.py index f35a2b9f9..6fdd7e302 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -4,7 +4,7 @@ from unittest import mock from warnings import catch_warnings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, BaseItem, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) @@ -334,29 +334,73 @@ class DictItemTest(unittest.TestCase): class BaseItemTest(unittest.TestCase): + def test_isinstance_check(self): + + class SubclassedBaseItem(BaseItem): + pass + + class SubclassedItem(Item): + pass + + self.assertTrue(isinstance(BaseItem(), BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) + self.assertTrue(isinstance(Item(), BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), BaseItem)) + + # make sure internal checks using private _BaseItem class succeed + self.assertTrue(isinstance(BaseItem(), _BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) + self.assertTrue(isinstance(Item(), _BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) + def test_deprecation_warning(self): + """ + Make sure deprecation warnings are logged whenever BaseItem is used, + either instantiated or in an isinstance check + """ with catch_warnings(record=True) as warnings: BaseItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + class SubclassedBaseItem(BaseItem): pass + SubclassedBaseItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + self.assertFalse(isinstance("foo", BaseItem)) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + + with catch_warnings(record=True) as warnings: + self.assertTrue(isinstance(BaseItem(), BaseItem)) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + class ItemNoDeprecationWarningTest(unittest.TestCase): - def test_no_deprecation_warning(self): + """ + Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used. + """ + class SubclassedItem(Item): + pass + with catch_warnings(record=True) as warnings: Item() - self.assertEqual(len(warnings), 0) - with catch_warnings(record=True) as warnings: - class SubclassedItem(Item): - pass SubclassedItem() + _BaseItem() + self.assertFalse(isinstance("foo", _BaseItem)) + self.assertFalse(isinstance("foo", Item)) + self.assertFalse(isinstance("foo", SubclassedItem)) + self.assertTrue(isinstance(_BaseItem(), _BaseItem)) + self.assertTrue(isinstance(Item(), Item)) + self.assertTrue(isinstance(SubclassedItem(), SubclassedItem)) self.assertEqual(len(warnings), 0) From f689e917bf2caa2c48def7399317dd019264663d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 8 May 2020 19:44:05 -0300 Subject: [PATCH 13/20] Update Item docstring --- scrapy/item.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index f468ff86f..b9a0acd3d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -79,8 +79,7 @@ class DictItem(MutableMapping, BaseItem): def __new__(cls, *args, **kwargs): if issubclass(cls, DictItem) and not issubclass(cls, Item): - warn('scrapy.item.DictItem is deprecated, please use ' - 'scrapy.item.Item instead', + warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) return super(DictItem, cls).__new__(cls, *args, **kwargs) @@ -147,6 +146,11 @@ class Item(DictItem, metaclass=ItemMeta): If you need instances of a custom class to be considered items by Scrapy, you must inherit from either :class:`Item` or :class:`dict`. + Items offer the ability to declare :class:`Field` attributes, which can be + used to define metadata and control the way data is processed internally. + Please refer to the :ref:`documentation about fields <topics-items-fields>` + for additional information. + Unlike instances of :class:`dict`, instances of :class:`Item` may be :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. """ From abfdc1b5425997f1aa69b29465c6ab2324f37fd0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Mon, 11 May 2020 13:20:06 -0300 Subject: [PATCH 14/20] Update docstring for Item class --- scrapy/item.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index b9a0acd3d..97dfed976 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -138,18 +138,20 @@ class Item(DictItem, metaclass=ItemMeta): """ Base class for scraped items. - In Scrapy, an object is considered an *item* if it is an instance of either - :class:`Item` or :class:`dict`. For example, when the output of a + In Scrapy, an object is considered an ``item`` if it is an instance of either + :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a spider callback is evaluated, only instances of :class:`Item` or :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`. If you need instances of a custom class to be considered items by Scrapy, you must inherit from either :class:`Item` or :class:`dict`. - Items offer the ability to declare :class:`Field` attributes, which can be - used to define metadata and control the way data is processed internally. - Please refer to the :ref:`documentation about fields <topics-items-fields>` - for additional information. + Items must declare :class:`Field` attributes, which are processed and stored + in the ``fields`` attribute. This restricts the set of allowed field names + and prevents typos, raising ``KeyError`` when referring to undefined fields. + Additionally, fields can be used to define metadata and control the way + data is processed internally. Please refer to the :ref:`documentation + about fields <topics-items-fields>` for additional information. Unlike instances of :class:`dict`, instances of :class:`Item` may be :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. From fffb0a5b6a7eaf365ecd7ef43e7e45cf2ea7ff2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com> Date: Fri, 15 May 2020 15:23:09 +0200 Subject: [PATCH 15/20] replace set constructor with set literals (#4573) --- tests/test_engine.py | 8 ++++---- tests/test_http_request.py | 6 +++--- tests/test_pipeline_crawl.py | 4 ++-- tests/test_pipeline_images.py | 2 +- tests/test_spiderloader/__init__.py | 8 ++++---- tests/test_utils_datatypes.py | 2 +- tests/test_utils_misc/__init__.py | 8 ++++---- tests/test_utils_url.py | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index acfe94f63..d781665dc 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -218,8 +218,8 @@ class EngineTest(unittest.TestCase): def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] - urls_visited = set([rp[0].url for rp in self.run.respplug]) - urls_expected = set([self.run.geturl(p) for p in must_be_visited]) + urls_visited = {rp[0].url for rp in self.run.respplug} + urls_expected = {self.run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) def _assert_scheduled_requests(self, urls_to_visit=None): @@ -227,8 +227,8 @@ class EngineTest(unittest.TestCase): paths_expected = ['/item999.html', '/item2.html', '/item1.html'] - urls_requested = set([rq[0].url for rq in self.run.reqplug]) - urls_expected = set([self.run.geturl(p) for p in paths_expected]) + urls_requested = {rq[0].url for rq in self.run.reqplug} + urls_expected = {self.run.geturl(p) for p in paths_expected} assert urls_expected <= urls_requested scheduled_requests_count = len(self.run.reqplug) dropped_requests_count = len(self.run.reqdropped) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a672963f3..349c23f36 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -549,8 +549,8 @@ class FormRequestTest(RequestTest): self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") fs = _qs(r1) - self.assertEqual(set(fs[b'test']), set([b'val1', b'val2'])) - self.assertEqual(set(fs[b'one']), set([b'two', b'three'])) + self.assertEqual(set(fs[b'test']), {b'val1', b'val2'}) + self.assertEqual(set(fs[b'one']), {b'two', b'three'}) self.assertEqual(fs[b'test2'], [b'xxx']) self.assertEqual(fs[b'six'], [b'seven']) @@ -1047,7 +1047,7 @@ class FormRequestTest(RequestTest): </form>''') req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(set(fs), set([b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'])) + self.assertEqual(set(fs), {b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'}) def test_from_response_xpath(self): response = _buildresponse( diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 6d15aaf31..74263e50c 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -51,10 +51,10 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = 'FILES_STORE' media_key = 'files' media_urls_key = 'file_urls' - expected_checksums = set([ + expected_checksums = { '5547178b89448faf0015a13f904c936e', 'c2281c83670e31d8aaab7cb642b824db', - 'ed3f6538dc15d4d9179dae57319edc5f']) + 'ed3f6538dc15d4d9179dae57319edc5f'} def setUp(self): self.mockserver = MockServer() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 5ba03ff4c..76aea470f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -18,7 +18,7 @@ try: except ImportError: skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' else: - encoders = set(('jpeg_encoder', 'jpeg_decoder')) + encoders = {'jpeg_encoder', 'jpeg_decoder'} if not encoders.issubset(set(Image.core.__dict__)): skip = 'Missing JPEG encoders' diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 265970b43..d922c6059 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -42,7 +42,7 @@ class SpiderLoaderTest(unittest.TestCase): def test_list(self): self.assertEqual( set(self.spider_loader.list()), - set(['spider1', 'spider2', 'spider3', 'spider4'])) + {'spider1', 'spider2', 'spider3', 'spider4'}) def test_load(self): spider1 = self.spider_loader.load("spider1") @@ -57,7 +57,7 @@ class SpiderLoaderTest(unittest.TestCase): ['spider2']) self.assertEqual( set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), - set(['spider1', 'spider2'])) + {'spider1', 'spider2'}) self.assertEqual( self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), []) @@ -151,7 +151,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) - self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) + self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'}) def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name @@ -177,4 +177,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) - self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) + self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'}) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index e5aa56eb9..0a4c6034a 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -217,7 +217,7 @@ class SequenceExcludeTest(unittest.TestCase): def test_set(self): """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" - seq = set([-3, "test", 1.1]) + seq = {-3, "test", 1.1} d = SequenceExclude(seq) self.assertIn(0, d) self.assertIn("foo", d) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 28205e0d9..9bb996d27 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -26,20 +26,20 @@ class UtilsMiscTestCase(unittest.TestCase): 'tests.test_utils_misc.test_walk_modules.mod.mod0', 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEqual(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual({m.__name__ for m in mods}, set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod') expected = [ 'tests.test_utils_misc.test_walk_modules.mod', 'tests.test_utils_misc.test_walk_modules.mod.mod0', ] - self.assertEqual(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual({m.__name__ for m in mods}, set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1') expected = [ 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEqual(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual({m.__name__ for m in mods}, set(expected)) self.assertRaises(ImportError, walk_modules, 'nomodule999') @@ -54,7 +54,7 @@ class UtilsMiscTestCase(unittest.TestCase): 'testegg.spiders.b', 'testegg' ] - self.assertEqual(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual({m.__name__ for m in mods}, set(expected)) finally: sys.path.remove(egg) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 16e7449c9..4ab944949 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -55,7 +55,7 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) - spider = Spider(name='example.com', allowed_domains=set(('example.com', 'example.net'))) + spider = Spider(name='example.com', allowed_domains={'example.com', 'example.net'}) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) spider = Spider(name='example.com', allowed_domains=('example.com', 'example.net')) From 36c3c9713e59f5d22bf51354920b5093e2d30b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Fri, 15 May 2020 19:37:56 +0200 Subject: [PATCH 16/20] Run tests with Python 3.5.0 (#4518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Run tests with Python 3.5.0 * Use Ubuntu Trusty 14.04 to test Python 3.5.0 * Use mitmproxy<3.0.0 on Python 3.5.0 * Skip tests requiring mitmproxy in Python 3.5.0 * Change the minimum Python version from 3.5 to 3.5.1 * Do not prevent Scrapy from working with Python 3.5.0 * Force system Python 3.5.1 * Do not install a non-system Python in Python 3.5.1 jobs * Switch to Trusty to be able to test Xenial’s Python version ¯\_(ツ)_/¯ * Add missing trusty * Stop breaking old PyPy * Allow installing Scrapy on Python 3.5.0 --- .travis.yml | 10 +++++++--- README.rst | 2 +- docs/faq.rst | 2 +- docs/intro/install.rst | 2 +- tests/test_proxy_connect.py | 5 ++++- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcbf75a43..d6ec88e06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,12 +17,16 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=pypy3 + - env: TOXENV=pinned + python: 3.5.1 + dist: trusty + - env: TOXENV=asyncio + python: 3.5.1 # We use additional code to support 3.5.3 and earlier + dist: trusty - env: TOXENV=py python: 3.5 - - env: TOXENV=pinned - python: 3.5 - env: TOXENV=asyncio - python: 3.5.2 + python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - env: TOXENV=py python: 3.6 - env: TOXENV=py diff --git a/README.rst b/README.rst index ce5973bcd..fd84e127e 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ including a list of features. Requirements ============ -* Python 3.5+ +* Python 3.5.1+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/faq.rst b/docs/faq.rst index 75a0f4864..936f315b3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 3.5+ +Scrapy is supported under Python 3.5.1+ under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). Python 3 support was added in Scrapy 1.1. PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 6356e0eea..4af80d801 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,7 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 3.5 or above under CPython (default Python +Scrapy runs on Python 3.5.1 or above under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 4763a5417..eb4ecc91d 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -4,6 +4,7 @@ import re import sys from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit +from unittest import skipIf import pytest from testfixtures import LogCapture @@ -56,6 +57,8 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) +@skipIf(sys.version_info < (3, 5, 4), + "requires mitmproxy < 3.0.0, which these tests do not support") class ProxyConnectTestCase(TestCase): def setUp(self): @@ -80,7 +83,7 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, log) - @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) + @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6)) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) From bcc40c40771ad223471ab77ad47233c498312095 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Mon, 18 May 2020 22:12:31 +0500 Subject: [PATCH 17/20] better deprecation warning for Response.body_as_unicode() (#4579) --- scrapy/http/response/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5614e6e55..0603b6653 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -65,7 +65,7 @@ class TextResponse(Response): """Return body as unicode""" warnings.warn('Response.body_as_unicode() is deprecated, ' 'please use Response.text instead.', - ScrapyDeprecationWarning) + ScrapyDeprecationWarning, stacklevel=2) return self.text @property From afb2e5011fb9d9aa0ca568d376b0c4cef4193a9e Mon Sep 17 00:00:00 2001 From: nsirletti <n.sirletti@gmail.com> Date: Sat, 16 May 2020 12:48:21 +0200 Subject: [PATCH 18/20] Allow doc to be downloadable on readthedocs.org --- .readthedocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 17eba34f3..e4d3f02cc 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,5 @@ version: 2 +formats: all sphinx: configuration: docs/conf.py fail_on_warning: true From c2a0cca0fe6bc8342efa6034a78b9b8161aa2177 Mon Sep 17 00:00:00 2001 From: nsirletti <n.sirletti@gmail.com> Date: Sat, 16 May 2020 15:07:16 +0200 Subject: [PATCH 19/20] Suppress Sphinx warnings about MIME types other than .rst --- docs/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 8ab38a090..468c1d190 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -100,6 +100,9 @@ exclude_trees = ['.build'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' +# List of Sphinx warnings that will not be raised +suppress_warnings = ['epub.unknown_project_files'] + # Options for HTML output # ----------------------- From f418c6d5f7307c3beae57dfc519be04657a8e1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 19 May 2020 20:04:47 +0200 Subject: [PATCH 20/20] logging.rst: remove unused, misleading import --- docs/topics/logging.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 675e65ef1..e81091651 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -262,7 +262,6 @@ scrapy.utils.log module This is an example on how to redirect ``INFO`` or higher messages to a file:: import logging - from scrapy.utils.log import configure_logging logging.basicConfig( filename='log.txt',